diff --git "a/dashq_config.json" "b/dashq_config.json" new file mode 100644--- /dev/null +++ "b/dashq_config.json" @@ -0,0 +1,250000 @@ +{ + "base_model": "Qwen/Qwen3.5-35B-A3B", + "format": "dashq-packed-linear", + "format_version": 1, + "method": "dashq", + "model_class": "image_text_to_text", + "params": { + "bits": 3, + "group_size": 128, + "low_memory_optimization": false, + "moe_hessian_scope": "shared", + "n_samples": 128, + "scale_zero_dtype": "float16", + "symmetric": false, + "use_error_compensation": true, + "use_optimal_shrinkage": true, + "use_weighted_quantization": true + }, + "quantized_modules": { + "model.language_model.layers.0.linear_attn.in_proj_a": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 512, + "out_features": 32, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.linear_attn.in_proj_b": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 512, + "out_features": 32, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.linear_attn.in_proj_qkv": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 131072, + "out_features": 8192, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.linear_attn.in_proj_z": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 65536, + "out_features": 4096, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.linear_attn.out_proj": { + "group_size": 128, + "in_features": 4096, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 65536, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 4096, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.0": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.1": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.10": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.100": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.101": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.102": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.103": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.104": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.105": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.106": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.107": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.108": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.109": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.11": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.110": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.111": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.112": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.113": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.114": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.115": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.116": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.117": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.118": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.119": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.12": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.120": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.121": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.122": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.123": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.124": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.125": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.126": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.127": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.128": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.129": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.13": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.130": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.131": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.132": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.133": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.134": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.135": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.136": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.137": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.138": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.139": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.14": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.140": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.141": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.142": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.143": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.144": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.145": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.146": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.147": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.148": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.149": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.15": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.150": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.151": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.152": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.153": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.154": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.155": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.156": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.157": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.158": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.159": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.16": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.160": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.161": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.162": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.163": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.164": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.165": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.166": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.167": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.168": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.169": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.17": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.170": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.171": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.172": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.173": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.174": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.175": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.176": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.177": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.178": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.179": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.18": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.180": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.181": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.182": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.183": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.184": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.185": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.186": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.187": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.188": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.189": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.19": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.190": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.191": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.192": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.193": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.194": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.195": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.196": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.197": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.198": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.199": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.2": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.20": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.200": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.201": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.202": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.203": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.204": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.205": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.206": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.207": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.208": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.209": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.21": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.210": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.211": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.212": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.213": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.214": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.215": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.216": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.217": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.218": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.219": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.22": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.220": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.221": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.222": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.223": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.224": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.225": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.226": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.227": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.228": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.229": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.23": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.230": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.231": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.232": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.233": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.234": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.235": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.236": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.237": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.238": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.239": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.24": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.240": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.241": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.242": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.243": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.244": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.245": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.246": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.247": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.248": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.249": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.25": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.250": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.251": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.252": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.253": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.254": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.255": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.26": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.27": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.28": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.29": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.3": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.30": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.31": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.32": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.33": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.34": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.35": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.36": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.37": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.38": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.39": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.4": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.40": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.41": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.42": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.43": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.44": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.45": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.46": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.47": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.48": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.49": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.5": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.50": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.51": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.52": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.53": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.54": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.55": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.56": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.57": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.58": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.59": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.6": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.60": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.61": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.62": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.63": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.64": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.65": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.66": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.67": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.68": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.69": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.7": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.70": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.71": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.72": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.73": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.74": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.75": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.76": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.77": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.78": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.79": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.8": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.80": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.81": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.82": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.83": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.84": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.85": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.86": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.87": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.88": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.89": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.9": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.90": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.91": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.92": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.93": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.94": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.95": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.96": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.97": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.98": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.down_proj_list.99": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.0": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.1": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.10": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.100": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.101": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.102": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.103": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.104": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.105": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.106": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.107": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.108": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.109": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.11": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.110": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.111": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.112": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.113": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.114": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.115": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.116": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.117": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.118": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.119": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.12": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.120": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.121": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.122": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.123": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.124": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.125": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.126": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.127": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.128": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.129": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.13": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.130": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.131": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.132": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.133": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.134": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.135": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.136": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.137": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.138": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.139": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.14": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.140": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.141": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.142": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.143": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.144": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.145": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.146": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.147": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.148": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.149": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.15": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.150": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.151": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.152": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.153": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.154": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.155": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.156": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.157": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.158": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.159": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.16": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.160": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.161": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.162": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.163": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.164": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.165": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.166": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.167": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.168": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.169": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.17": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.170": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.171": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.172": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.173": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.174": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.175": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.176": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.177": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.178": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.179": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.18": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.180": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.181": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.182": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.183": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.184": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.185": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.186": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.187": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.188": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.189": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.19": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.190": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.191": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.192": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.193": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.194": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.195": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.196": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.197": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.198": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.199": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.2": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.20": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.200": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.201": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.202": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.203": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.204": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.205": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.206": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.207": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.208": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.209": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.21": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.210": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.211": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.212": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.213": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.214": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.215": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.216": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.217": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.218": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.219": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.22": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.220": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.221": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.222": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.223": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.224": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.225": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.226": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.227": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.228": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.229": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.23": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.230": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.231": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.232": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.233": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.234": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.235": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.236": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.237": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.238": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.239": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.24": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.240": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.241": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.242": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.243": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.244": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.245": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.246": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.247": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.248": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.249": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.25": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.250": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.251": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.252": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.253": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.254": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.255": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.26": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.27": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.28": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.29": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.3": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.30": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.31": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.32": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.33": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.34": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.35": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.36": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.37": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.38": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.39": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.4": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.40": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.41": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.42": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.43": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.44": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.45": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.46": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.47": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.48": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.49": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.5": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.50": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.51": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.52": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.53": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.54": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.55": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.56": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.57": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.58": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.59": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.6": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.60": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.61": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.62": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.63": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.64": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.65": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.66": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.67": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.68": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.69": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.7": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.70": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.71": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.72": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.73": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.74": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.75": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.76": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.77": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.78": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.79": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.8": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.80": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.81": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.82": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.83": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.84": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.85": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.86": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.87": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.88": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.89": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.9": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.90": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.91": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.92": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.93": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.94": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.95": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.96": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.97": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.98": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.experts.gate_up_proj_list.99": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.shared_expert.down_proj": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.shared_expert.gate_proj": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 512, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.shared_expert.up_proj": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 512, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.0.mlp.shared_expert_gate": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16, + "out_features": 1, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.linear_attn.in_proj_a": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 512, + "out_features": 32, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.linear_attn.in_proj_b": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 512, + "out_features": 32, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.linear_attn.in_proj_qkv": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 131072, + "out_features": 8192, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.linear_attn.in_proj_z": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 65536, + "out_features": 4096, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.linear_attn.out_proj": { + "group_size": 128, + "in_features": 4096, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 65536, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 4096, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.0": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.1": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.10": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.100": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.101": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.102": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.103": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.104": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.105": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.106": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.107": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.108": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.109": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.11": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.110": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.111": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.112": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.113": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.114": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.115": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.116": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.117": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.118": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.119": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.12": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.120": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.121": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.122": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.123": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.124": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.125": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.126": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.127": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.128": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.129": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.13": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.130": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.131": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.132": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.133": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.134": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.135": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.136": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.137": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.138": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.139": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.14": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.140": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.141": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.142": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.143": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.144": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.145": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.146": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.147": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.148": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.149": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.15": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.150": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.151": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.152": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.153": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.154": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.155": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.156": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.157": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.158": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.159": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.16": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.160": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.161": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.162": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.163": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.164": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.165": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.166": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.167": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.168": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.169": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.17": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.170": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.171": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.172": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.173": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.174": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.175": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.176": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.177": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.178": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.179": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.18": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.180": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.181": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.182": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.183": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.184": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.185": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.186": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.187": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.188": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.189": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.19": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.190": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.191": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.192": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.193": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.194": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.195": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.196": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.197": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.198": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.199": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.2": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.20": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.200": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.201": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.202": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.203": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.204": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.205": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.206": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.207": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.208": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.209": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.21": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.210": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.211": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.212": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.213": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.214": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.215": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.216": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.217": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.218": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.219": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.22": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.220": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.221": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.222": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.223": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.224": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.225": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.226": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.227": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.228": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.229": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.23": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.230": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.231": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.232": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.233": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.234": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.235": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.236": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.237": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.238": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.239": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.24": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.240": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.241": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.242": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.243": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.244": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.245": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.246": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.247": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.248": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.249": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.25": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.250": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.251": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.252": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.253": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.254": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.255": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.26": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.27": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.28": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.29": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.3": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.30": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.31": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.32": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.33": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.34": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.35": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.36": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.37": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.38": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.39": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.4": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.40": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.41": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.42": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.43": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.44": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.45": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.46": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.47": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.48": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.49": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.5": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.50": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.51": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.52": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.53": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.54": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.55": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.56": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.57": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.58": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.59": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.6": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.60": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.61": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.62": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.63": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.64": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.65": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.66": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.67": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.68": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.69": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.7": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.70": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.71": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.72": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.73": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.74": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.75": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.76": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.77": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.78": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.79": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.8": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.80": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.81": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.82": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.83": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.84": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.85": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.86": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.87": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.88": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.89": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.9": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.90": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.91": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.92": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.93": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.94": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.95": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.96": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.97": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.98": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.down_proj_list.99": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.0": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.1": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.10": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.100": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.101": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.102": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.103": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.104": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.105": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.106": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.107": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.108": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.109": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.11": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.110": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.111": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.112": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.113": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.114": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.115": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.116": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.117": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.118": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.119": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.12": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.120": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.121": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.122": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.123": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.124": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.125": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.126": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.127": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.128": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.129": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.13": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.130": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.131": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.132": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.133": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.134": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.135": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.136": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.137": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.138": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.139": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.14": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.140": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.141": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.142": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.143": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.144": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.145": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.146": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.147": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.148": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.149": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.15": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.150": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.151": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.152": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.153": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.154": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.155": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.156": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.157": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.158": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.159": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.16": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.160": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.161": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.162": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.163": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.164": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.165": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.166": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.167": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.168": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.169": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.17": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.170": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.171": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.172": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.173": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.174": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.175": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.176": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.177": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.178": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.179": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.18": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.180": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.181": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.182": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.183": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.184": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.185": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.186": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.187": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.188": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.189": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.19": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.190": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.191": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.192": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.193": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.194": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.195": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.196": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.197": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.198": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.199": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.2": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.20": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.200": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.201": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.202": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.203": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.204": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.205": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.206": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.207": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.208": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.209": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.21": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.210": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.211": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.212": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.213": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.214": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.215": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.216": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.217": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.218": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.219": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.22": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.220": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.221": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.222": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.223": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.224": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.225": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.226": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.227": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.228": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.229": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.23": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.230": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.231": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.232": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.233": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.234": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.235": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.236": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.237": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.238": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.239": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.24": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.240": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.241": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.242": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.243": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.244": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.245": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.246": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.247": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.248": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.249": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.25": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.250": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.251": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.252": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.253": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.254": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.255": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.26": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.27": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.28": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.29": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.3": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.30": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.31": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.32": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.33": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.34": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.35": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.36": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.37": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.38": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.39": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.4": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.40": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.41": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.42": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.43": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.44": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.45": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.46": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.47": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.48": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.49": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.5": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.50": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.51": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.52": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.53": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.54": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.55": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.56": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.57": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.58": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.59": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.6": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.60": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.61": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.62": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.63": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.64": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.65": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.66": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.67": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.68": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.69": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.7": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.70": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.71": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.72": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.73": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.74": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.75": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.76": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.77": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.78": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.79": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.8": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.80": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.81": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.82": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.83": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.84": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.85": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.86": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.87": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.88": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.89": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.9": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.90": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.91": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.92": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.93": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.94": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.95": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.96": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.97": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.98": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.experts.gate_up_proj_list.99": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.shared_expert.down_proj": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.shared_expert.gate_proj": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 512, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.shared_expert.up_proj": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 512, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.1.mlp.shared_expert_gate": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16, + "out_features": 1, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.linear_attn.in_proj_a": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 512, + "out_features": 32, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.linear_attn.in_proj_b": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 512, + "out_features": 32, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.linear_attn.in_proj_qkv": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 131072, + "out_features": 8192, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.linear_attn.in_proj_z": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 65536, + "out_features": 4096, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.linear_attn.out_proj": { + "group_size": 128, + "in_features": 4096, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 65536, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 4096, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.0": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.1": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.10": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.100": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.101": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.102": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.103": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.104": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.105": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.106": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.107": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.108": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.109": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.11": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.110": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.111": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.112": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.113": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.114": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.115": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.116": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.117": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.118": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.119": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.12": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.120": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.121": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.122": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.123": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.124": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.125": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.126": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.127": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.128": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.129": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.13": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.130": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.131": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.132": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.133": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.134": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.135": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.136": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.137": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.138": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.139": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.14": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.140": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.141": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.142": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.143": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.144": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.145": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.146": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.147": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.148": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.149": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.15": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.150": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.151": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.152": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.153": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.154": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.155": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.156": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.157": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.158": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.159": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.16": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.160": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.161": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.162": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.163": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.164": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.165": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.166": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.167": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.168": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.169": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.17": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.170": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.171": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.172": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.173": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.174": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.175": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.176": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.177": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.178": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.179": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.18": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.180": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.181": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.182": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.183": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.184": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.185": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.186": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.187": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.188": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.189": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.19": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.190": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.191": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.192": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.193": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.194": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.195": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.196": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.197": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.198": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.199": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.2": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.20": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.200": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.201": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.202": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.203": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.204": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.205": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.206": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.207": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.208": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.209": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.21": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.210": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.211": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.212": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.213": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.214": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.215": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.216": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.217": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.218": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.219": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.22": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.220": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.221": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.222": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.223": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.224": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.225": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.226": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.227": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.228": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.229": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.23": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.230": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.231": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.232": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.233": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.234": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.235": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.236": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.237": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.238": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.239": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.24": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.240": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.241": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.242": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.243": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.244": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.245": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.246": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.247": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.248": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.249": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.25": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.250": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.251": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.252": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.253": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.254": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.255": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.26": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.27": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.28": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.29": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.3": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.30": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.31": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.32": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.33": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.34": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.35": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.36": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.37": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.38": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.39": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.4": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.40": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.41": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.42": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.43": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.44": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.45": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.46": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.47": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.48": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.49": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.5": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.50": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.51": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.52": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.53": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.54": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.55": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.56": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.57": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.58": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.59": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.6": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.60": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.61": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.62": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.63": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.64": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.65": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.66": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.67": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.68": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.69": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.7": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.70": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.71": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.72": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.73": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.74": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.75": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.76": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.77": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.78": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.79": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.8": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.80": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.81": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.82": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.83": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.84": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.85": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.86": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.87": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.88": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.89": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.9": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.90": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.91": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.92": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.93": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.94": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.95": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.96": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.97": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.98": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.down_proj_list.99": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.0": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.1": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.10": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.100": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.101": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.102": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.103": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.104": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.105": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.106": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.107": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.108": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.109": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.11": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.110": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.111": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.112": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.113": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.114": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.115": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.116": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.117": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.118": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.119": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.12": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.120": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.121": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.122": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.123": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.124": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.125": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.126": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.127": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.128": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.129": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.13": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.130": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.131": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.132": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.133": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.134": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.135": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.136": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.137": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.138": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.139": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.14": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.140": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.141": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.142": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.143": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.144": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.145": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.146": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.147": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.148": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.149": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.15": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.150": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.151": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.152": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.153": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.154": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.155": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.156": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.157": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.158": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.159": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.16": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.160": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.161": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.162": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.163": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.164": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.165": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.166": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.167": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.168": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.169": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.17": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.170": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.171": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.172": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.173": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.174": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.175": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.176": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.177": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.178": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.179": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.18": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.180": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.181": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.182": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.183": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.184": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.185": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.186": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.187": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.188": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.189": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.19": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.190": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.191": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.192": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.193": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.194": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.195": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.196": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.197": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.198": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.199": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.2": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.20": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.200": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.201": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.202": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.203": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.204": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.205": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.206": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.207": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.208": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.209": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.21": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.210": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.211": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.212": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.213": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.214": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.215": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.216": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.217": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.218": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.219": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.22": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.220": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.221": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.222": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.223": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.224": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.225": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.226": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.227": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.228": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.229": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.23": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.230": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.231": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.232": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.233": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.234": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.235": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.236": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.237": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.238": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.239": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.24": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.240": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.241": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.242": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.243": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.244": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.245": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.246": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.247": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.248": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.249": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.25": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.250": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.251": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.252": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.253": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.254": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.255": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.26": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.27": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.28": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.29": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.3": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.30": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.31": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.32": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.33": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.34": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.35": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.36": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.37": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.38": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.39": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.4": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.40": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.41": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.42": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.43": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.44": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.45": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.46": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.47": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.48": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.49": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.5": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.50": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.51": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.52": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.53": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.54": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.55": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.56": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.57": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.58": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.59": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.6": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.60": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.61": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.62": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.63": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.64": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.65": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.66": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.67": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.68": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.69": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.7": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.70": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.71": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.72": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.73": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.74": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.75": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.76": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.77": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.78": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.79": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.8": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.80": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.81": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.82": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.83": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.84": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.85": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.86": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.87": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.88": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.89": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.9": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.90": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.91": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.92": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.93": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.94": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.95": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.96": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.97": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.98": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.experts.gate_up_proj_list.99": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.shared_expert.down_proj": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.shared_expert.gate_proj": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 512, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.shared_expert.up_proj": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 512, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.10.mlp.shared_expert_gate": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16, + "out_features": 1, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.0": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.1": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.10": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.100": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.101": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.102": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.103": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.104": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.105": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.106": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.107": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.108": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.109": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.11": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.110": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.111": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.112": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.113": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.114": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.115": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.116": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.117": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.118": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.119": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.12": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.120": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.121": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.122": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.123": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.124": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.125": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.126": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.127": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.128": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.129": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.13": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.130": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.131": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.132": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.133": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.134": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.135": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.136": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.137": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.138": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.139": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.14": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.140": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.141": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.142": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.143": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.144": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.145": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.146": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.147": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.148": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.149": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.15": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.150": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.151": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.152": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.153": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.154": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.155": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.156": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.157": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.158": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.159": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.16": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.160": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.161": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.162": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.163": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.164": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.165": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.166": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.167": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.168": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.169": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.17": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.170": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.171": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.172": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.173": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.174": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.175": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.176": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.177": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.178": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.179": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.18": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.180": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.181": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.182": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.183": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.184": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.185": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.186": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.187": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.188": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.189": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.19": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.190": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.191": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.192": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.193": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.194": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.195": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.196": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.197": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.198": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.199": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.2": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.20": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.200": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.201": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.202": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.203": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.204": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.205": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.206": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.207": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.208": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.209": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.21": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.210": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.211": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.212": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.213": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.214": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.215": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.216": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.217": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.218": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.219": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.22": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.220": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.221": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.222": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.223": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.224": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.225": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.226": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.227": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.228": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.229": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.23": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.230": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.231": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.232": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.233": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.234": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.235": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.236": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.237": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.238": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.239": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.24": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.240": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.241": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.242": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.243": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.244": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.245": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.246": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.247": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.248": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.249": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.25": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.250": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.251": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.252": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.253": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.254": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.255": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.26": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.27": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.28": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.29": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.3": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.30": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.31": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.32": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.33": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.34": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.35": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.36": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.37": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.38": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.39": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.4": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.40": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.41": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.42": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.43": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.44": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.45": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.46": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.47": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.48": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.49": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.5": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.50": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.51": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.52": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.53": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.54": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.55": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.56": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.57": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.58": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.59": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.6": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.60": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.61": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.62": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.63": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.64": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.65": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.66": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.67": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.68": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.69": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.7": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.70": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.71": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.72": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.73": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.74": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.75": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.76": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.77": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.78": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.79": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.8": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.80": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.81": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.82": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.83": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.84": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.85": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.86": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.87": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.88": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.89": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.9": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.90": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.91": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.92": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.93": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.94": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.95": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.96": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.97": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.98": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.down_proj_list.99": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.0": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.1": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.10": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.100": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.101": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.102": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.103": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.104": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.105": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.106": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.107": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.108": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.109": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.11": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.110": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.111": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.112": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.113": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.114": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.115": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.116": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.117": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.118": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.119": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.12": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.120": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.121": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.122": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.123": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.124": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.125": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.126": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.127": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.128": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.129": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.13": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.130": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.131": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.132": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.133": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.134": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.135": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.136": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.137": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.138": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.139": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.14": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.140": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.141": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.142": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.143": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.144": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.145": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.146": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.147": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.148": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.149": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.15": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.150": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.151": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.152": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.153": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.154": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.155": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.156": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.157": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.158": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.159": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.16": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.160": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.161": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.162": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.163": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.164": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.165": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.166": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.167": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.168": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.169": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.17": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.170": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.171": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.172": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.173": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.174": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.175": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.176": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.177": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.178": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.179": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.18": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.180": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.181": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.182": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.183": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.184": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.185": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.186": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.187": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.188": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.189": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.19": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.190": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.191": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.192": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.193": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.194": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.195": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.196": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.197": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.198": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.199": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.2": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.20": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.200": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.201": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.202": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.203": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.204": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.205": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.206": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.207": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.208": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.209": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.21": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.210": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.211": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.212": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.213": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.214": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.215": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.216": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.217": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.218": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.219": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.22": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.220": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.221": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.222": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.223": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.224": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.225": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.226": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.227": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.228": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.229": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.23": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.230": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.231": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.232": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.233": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.234": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.235": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.236": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.237": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.238": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.239": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.24": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.240": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.241": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.242": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.243": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.244": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.245": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.246": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.247": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.248": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.249": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.25": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.250": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.251": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.252": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.253": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.254": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.255": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.26": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.27": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.28": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.29": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.3": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.30": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.31": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.32": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.33": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.34": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.35": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.36": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.37": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.38": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.39": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.4": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.40": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.41": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.42": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.43": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.44": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.45": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.46": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.47": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.48": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.49": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.5": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.50": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.51": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.52": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.53": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.54": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.55": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.56": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.57": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.58": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.59": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.6": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.60": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.61": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.62": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.63": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.64": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.65": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.66": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.67": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.68": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.69": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.7": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.70": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.71": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.72": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.73": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.74": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.75": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.76": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.77": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.78": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.79": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.8": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.80": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.81": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.82": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.83": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.84": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.85": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.86": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.87": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.88": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.89": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.9": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.90": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.91": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.92": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.93": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.94": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.95": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.96": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.97": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.98": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.experts.gate_up_proj_list.99": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.shared_expert.down_proj": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.shared_expert.gate_proj": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 512, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.shared_expert.up_proj": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 512, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.mlp.shared_expert_gate": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16, + "out_features": 1, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.self_attn.k_proj": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 512, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.self_attn.o_proj": { + "group_size": 128, + "in_features": 4096, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 65536, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 4096, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.self_attn.q_proj": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 131072, + "out_features": 8192, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.11.self_attn.v_proj": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 512, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.linear_attn.in_proj_a": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 512, + "out_features": 32, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.linear_attn.in_proj_b": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 512, + "out_features": 32, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.linear_attn.in_proj_qkv": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 131072, + "out_features": 8192, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.linear_attn.in_proj_z": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 65536, + "out_features": 4096, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.linear_attn.out_proj": { + "group_size": 128, + "in_features": 4096, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 65536, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 4096, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.0": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.1": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.10": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.100": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.101": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.102": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.103": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.104": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.105": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.106": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.107": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.108": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.109": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.11": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.110": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.111": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.112": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.113": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.114": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.115": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.116": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.117": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.118": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.119": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.12": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.120": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.121": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.122": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.123": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.124": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.125": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.126": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.127": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.128": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.129": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.13": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.130": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.131": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.132": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.133": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.134": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.135": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.136": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.137": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.138": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.139": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.14": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.140": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.141": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.142": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.143": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.144": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.145": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.146": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.147": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.148": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.149": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.15": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.150": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.151": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.152": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.153": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.154": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.155": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.156": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.157": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.158": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.159": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.16": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.160": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.161": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.162": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.163": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.164": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.165": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.166": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.167": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.168": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.169": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.17": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.170": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.171": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.172": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.173": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.174": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.175": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.176": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.177": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.178": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.179": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.18": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.180": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.181": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.182": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.183": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.184": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.185": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.186": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.187": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.188": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.189": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.19": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.190": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.191": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.192": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.193": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.194": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.195": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.196": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.197": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.198": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.199": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.2": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.20": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.200": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.201": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.202": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.203": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.204": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.205": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.206": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.207": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.208": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.209": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.21": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.210": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.211": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.212": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.213": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.214": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.215": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.216": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.217": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.218": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.219": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.22": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.220": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.221": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.222": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.223": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.224": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.225": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.226": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.227": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.228": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.229": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.23": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.230": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.231": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.232": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.233": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.234": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.235": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.236": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.237": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.238": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.239": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.24": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.240": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.241": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.242": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.243": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.244": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.245": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.246": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.247": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.248": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.249": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.25": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.250": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.251": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.252": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.253": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.254": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.255": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.26": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.27": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.28": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.29": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.3": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.30": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.31": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.32": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.33": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.34": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.35": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.36": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.37": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.38": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.39": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.4": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.40": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.41": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.42": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.43": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.44": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.45": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.46": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.47": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.48": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.49": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.5": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.50": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.51": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.52": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.53": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.54": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.55": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.56": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.57": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.58": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.59": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.6": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.60": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.61": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.62": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.63": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.64": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.65": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.66": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.67": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.68": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.69": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.7": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.70": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.71": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.72": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.73": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.74": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.75": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.76": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.77": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.78": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.79": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.8": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.80": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.81": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.82": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.83": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.84": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.85": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.86": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.87": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.88": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.89": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.9": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.90": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.91": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.92": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.93": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.94": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.95": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.96": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.97": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.98": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.down_proj_list.99": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.0": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.1": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.10": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.100": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.101": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.102": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.103": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.104": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.105": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.106": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.107": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.108": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.109": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.11": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.110": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.111": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.112": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.113": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.114": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.115": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.116": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.117": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.118": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.119": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.12": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.120": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.121": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.122": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.123": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.124": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.125": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.126": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.127": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.128": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.129": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.13": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.130": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.131": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.132": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.133": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.134": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.135": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.136": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.137": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.138": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.139": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.14": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.140": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.141": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.142": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.143": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.144": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.145": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.146": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.147": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.148": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.149": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.15": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.150": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.151": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.152": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.153": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.154": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.155": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.156": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.157": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.158": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.159": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.16": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.160": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.161": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.162": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.163": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.164": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.165": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.166": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.167": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.168": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.169": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.17": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.170": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.171": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.172": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.173": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.174": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.175": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.176": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.177": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.178": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.179": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.18": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.180": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.181": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.182": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.183": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.184": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.185": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.186": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.187": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.188": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.189": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.19": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.190": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.191": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.192": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.193": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.194": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.195": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.196": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.197": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.198": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.199": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.2": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.20": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.200": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.201": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.202": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.203": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.204": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.205": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.206": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.207": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.208": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.209": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.21": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.210": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.211": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.212": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.213": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.214": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.215": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.216": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.217": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.218": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.219": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.22": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.220": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.221": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.222": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.223": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.224": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.225": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.226": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.227": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.228": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.229": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.23": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.230": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.231": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.232": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.233": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.234": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.235": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.236": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.237": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.238": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.239": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.24": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.240": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.241": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.242": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.243": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.244": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.245": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.246": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.247": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.248": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.249": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.25": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.250": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.251": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.252": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.253": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.254": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.255": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.26": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.27": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.28": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.29": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.3": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.30": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.31": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.32": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.33": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.34": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.35": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.36": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.37": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.38": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.39": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.4": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.40": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.41": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.42": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.43": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.44": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.45": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.46": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.47": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.48": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.49": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.5": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.50": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.51": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.52": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.53": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.54": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.55": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.56": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.57": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.58": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.59": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.6": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.60": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.61": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.62": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.63": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.64": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.65": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.66": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.67": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.68": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.69": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.7": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.70": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.71": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.72": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.73": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.74": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.75": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.76": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.77": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.78": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.79": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.8": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.80": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.81": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.82": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.83": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.84": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.85": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.86": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.87": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.88": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.89": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.9": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.90": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.91": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.92": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.93": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.94": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.95": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.96": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.97": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.98": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.experts.gate_up_proj_list.99": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.shared_expert.down_proj": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.shared_expert.gate_proj": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 512, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.shared_expert.up_proj": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 512, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.12.mlp.shared_expert_gate": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16, + "out_features": 1, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.linear_attn.in_proj_a": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 512, + "out_features": 32, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.linear_attn.in_proj_b": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 512, + "out_features": 32, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.linear_attn.in_proj_qkv": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 131072, + "out_features": 8192, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.linear_attn.in_proj_z": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 65536, + "out_features": 4096, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.linear_attn.out_proj": { + "group_size": 128, + "in_features": 4096, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 65536, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 4096, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.0": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.1": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.10": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.100": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.101": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.102": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.103": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.104": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.105": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.106": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.107": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.108": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.109": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.11": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.110": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.111": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.112": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.113": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.114": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.115": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.116": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.117": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.118": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.119": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.12": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.120": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.121": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.122": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.123": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.124": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.125": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.126": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.127": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.128": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.129": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.13": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.130": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.131": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.132": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.133": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.134": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.135": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.136": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.137": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.138": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.139": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.14": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.140": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.141": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.142": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.143": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.144": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.145": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.146": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.147": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.148": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.149": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.15": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.150": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.151": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.152": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.153": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.154": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.155": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.156": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.157": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.158": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.159": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.16": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.160": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.161": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.162": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.163": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.164": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.165": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.166": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.167": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.168": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.169": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.17": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.170": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.171": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.172": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.173": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.174": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.175": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.176": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.177": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.178": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.179": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.18": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.180": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.181": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.182": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.183": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.184": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.185": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.186": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.187": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.188": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.189": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.19": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.190": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.191": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.192": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.193": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.194": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.195": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.196": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.197": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.198": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.199": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.2": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.20": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.200": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.201": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.202": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.203": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.204": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.205": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.206": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.207": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.208": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.209": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.21": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.210": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.211": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.212": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.213": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.214": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.215": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.216": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.217": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.218": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.219": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.22": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.220": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.221": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.222": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.223": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.224": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.225": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.226": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.227": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.228": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.229": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.23": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.230": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.231": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.232": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.233": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.234": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.235": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.236": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.237": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.238": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.239": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.24": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.240": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.241": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.242": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.243": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.244": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.245": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.246": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.247": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.248": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.249": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.25": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.250": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.251": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.252": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.253": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.254": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.255": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.26": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.27": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.28": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.29": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.3": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.30": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.31": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.32": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.33": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.34": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.35": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.36": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.37": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.38": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.39": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.4": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.40": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.41": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.42": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.43": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.44": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.45": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.46": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.47": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.48": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.49": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.5": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.50": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.51": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.52": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.53": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.54": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.55": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.56": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.57": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.58": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.59": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.6": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.60": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.61": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.62": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.63": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.64": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.65": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.66": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.67": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.68": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.69": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.7": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.70": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.71": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.72": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.73": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.74": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.75": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.76": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.77": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.78": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.79": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.8": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.80": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.81": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.82": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.83": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.84": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.85": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.86": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.87": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.88": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.89": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.9": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.90": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.91": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.92": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.93": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.94": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.95": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.96": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.97": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.98": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.down_proj_list.99": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.0": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.1": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.10": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.100": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.101": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.102": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.103": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.104": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.105": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.106": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.107": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.108": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.109": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.11": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.110": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.111": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.112": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.113": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.114": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.115": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.116": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.117": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.118": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.119": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.12": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.120": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.121": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.122": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.123": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.124": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.125": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.126": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.127": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.128": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.129": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.13": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.130": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.131": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.132": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.133": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.134": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.135": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.136": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.137": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.138": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.139": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.14": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.140": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.141": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.142": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.143": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.144": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.145": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.146": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.147": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.148": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.149": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.15": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.150": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.151": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.152": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.153": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.154": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.155": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.156": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.157": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.158": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.159": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.16": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.160": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.161": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.162": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.163": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.164": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.165": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.166": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.167": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.168": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.169": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.17": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.170": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.171": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.172": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.173": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.174": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.175": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.176": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.177": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.178": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.179": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.18": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.180": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.181": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.182": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.183": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.184": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.185": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.186": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.187": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.188": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.189": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.19": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.190": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.191": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.192": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.193": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.194": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.195": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.196": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.197": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.198": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.199": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.2": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.20": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.200": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.201": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.202": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.203": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.204": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.205": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.206": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.207": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.208": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.209": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.21": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.210": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.211": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.212": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.213": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.214": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.215": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.216": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.217": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.218": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.219": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.22": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.220": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.221": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.222": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.223": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.224": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.225": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.226": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.227": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.228": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.229": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.23": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.230": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.231": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.232": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.233": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.234": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.235": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.236": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.237": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.238": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.239": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.24": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.240": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.241": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.242": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.243": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.244": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.245": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.246": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.247": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.248": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.249": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.25": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.250": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.251": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.252": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.253": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.254": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.255": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.26": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.27": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.28": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.29": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.3": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.30": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.31": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.32": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.33": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.34": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.35": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.36": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.37": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.38": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.39": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.4": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.40": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.41": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.42": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.43": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.44": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.45": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.46": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.47": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.48": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.49": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.5": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.50": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.51": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.52": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.53": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.54": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.55": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.56": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.57": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.58": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.59": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.6": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.60": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.61": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.62": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.63": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.64": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.65": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.66": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.67": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.68": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.69": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.7": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.70": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.71": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.72": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.73": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.74": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.75": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.76": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.77": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.78": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.79": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.8": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.80": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.81": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.82": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.83": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.84": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.85": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.86": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.87": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.88": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.89": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.9": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.90": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.91": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.92": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.93": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.94": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.95": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.96": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.97": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.98": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.experts.gate_up_proj_list.99": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.shared_expert.down_proj": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.shared_expert.gate_proj": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 512, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.shared_expert.up_proj": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 512, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.13.mlp.shared_expert_gate": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16, + "out_features": 1, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.linear_attn.in_proj_a": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 512, + "out_features": 32, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.linear_attn.in_proj_b": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 512, + "out_features": 32, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.linear_attn.in_proj_qkv": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 131072, + "out_features": 8192, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.linear_attn.in_proj_z": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 65536, + "out_features": 4096, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.linear_attn.out_proj": { + "group_size": 128, + "in_features": 4096, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 65536, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 4096, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.0": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.1": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.10": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.100": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.101": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.102": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.103": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.104": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.105": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.106": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.107": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.108": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.109": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.11": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.110": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.111": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.112": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.113": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.114": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.115": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.116": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.117": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.118": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.119": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.12": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.120": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.121": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.122": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.123": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.124": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.125": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.126": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.127": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.128": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.129": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.13": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.130": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.131": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.132": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.133": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.134": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.135": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.136": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.137": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.138": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.139": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.14": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.140": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.141": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.142": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.143": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.144": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.145": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.146": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.147": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.148": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.149": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.15": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.150": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.151": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.152": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.153": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.154": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.155": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.156": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.157": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.158": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.159": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.16": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.160": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.161": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.162": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.163": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.164": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.165": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.166": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.167": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.168": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.169": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.17": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.170": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.171": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.172": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.173": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.174": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.175": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.176": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.177": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.178": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.179": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.18": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.180": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.181": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.182": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.183": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.184": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.185": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.186": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.187": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.188": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.189": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.19": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.190": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.191": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.192": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.193": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.194": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.195": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.196": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.197": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.198": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.199": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.2": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.20": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.200": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.201": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.202": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.203": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.204": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.205": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.206": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.207": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.208": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.209": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.21": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.210": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.211": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.212": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.213": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.214": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.215": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.216": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.217": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.218": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.219": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.22": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.220": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.221": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.222": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.223": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.224": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.225": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.226": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.227": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.228": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.229": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.23": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.230": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.231": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.232": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.233": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.234": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.235": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.236": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.237": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.238": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.239": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.24": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.240": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.241": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.242": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.243": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.244": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.245": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.246": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.247": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.248": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.249": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.25": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.250": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.251": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.252": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.253": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.254": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.255": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.26": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.27": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.28": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.29": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.3": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.30": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.31": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.32": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.33": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.34": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.35": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.36": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.37": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.38": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.39": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.4": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.40": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.41": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.42": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.43": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.44": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.45": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.46": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.47": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.48": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.49": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.5": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.50": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.51": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.52": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.53": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.54": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.55": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.56": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.57": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.58": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.59": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.6": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.60": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.61": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.62": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.63": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.64": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.65": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.66": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.67": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.68": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.69": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.7": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.70": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.71": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.72": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.73": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.74": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.75": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.76": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.77": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.78": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.79": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.8": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.80": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.81": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.82": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.83": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.84": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.85": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.86": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.87": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.88": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.89": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.9": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.90": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.91": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.92": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.93": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.94": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.95": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.96": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.97": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.98": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.down_proj_list.99": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.0": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.1": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.10": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.100": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.101": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.102": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.103": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.104": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.105": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.106": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.107": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.108": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.109": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.11": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.110": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.111": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.112": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.113": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.114": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.115": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.116": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.117": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.118": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.119": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.12": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.120": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.121": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.122": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.123": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.124": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.125": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.126": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.127": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.128": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.129": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.13": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.130": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.131": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.132": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.133": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.134": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.135": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.136": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.137": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.138": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.139": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.14": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.140": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.141": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.142": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.143": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.144": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.145": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.146": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.147": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.148": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.149": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.15": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.150": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.151": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.152": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.153": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.154": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.155": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.156": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.157": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.158": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.159": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.16": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.160": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.161": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.162": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.163": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.164": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.165": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.166": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.167": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.168": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.169": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.17": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.170": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.171": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.172": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.173": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.174": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.175": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.176": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.177": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.178": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.179": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.18": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.180": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.181": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.182": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.183": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.184": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.185": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.186": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.187": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.188": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.189": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.19": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.190": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.191": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.192": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.193": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.194": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.195": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.196": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.197": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.198": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.199": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.2": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.20": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.200": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.201": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.202": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.203": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.204": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.205": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.206": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.207": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.208": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.209": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.21": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.210": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.211": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.212": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.213": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.214": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.215": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.216": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.217": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.218": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.219": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.22": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.220": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.221": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.222": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.223": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.224": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.225": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.226": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.227": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.228": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.229": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.23": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.230": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.231": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.232": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.233": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.234": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.235": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.236": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.237": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.238": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.239": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.24": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.240": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.241": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.242": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.243": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.244": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.245": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.246": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.247": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.248": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.249": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.25": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.250": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.251": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.252": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.253": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.254": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.255": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.26": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.27": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.28": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.29": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.3": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.30": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.31": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.32": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.33": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.34": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.35": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.36": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.37": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.38": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.39": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.4": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.40": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.41": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.42": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.43": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.44": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.45": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.46": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.47": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.48": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.49": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.5": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.50": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.51": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.52": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.53": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.54": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.55": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.56": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.57": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.58": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.59": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.6": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.60": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.61": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.62": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.63": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.64": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.65": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.66": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.67": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.68": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.69": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.7": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.70": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.71": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.72": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.73": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.74": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.75": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.76": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.77": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.78": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.79": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.8": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.80": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.81": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.82": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.83": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.84": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.85": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.86": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.87": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.88": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.89": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.9": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.90": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.91": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.92": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.93": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.94": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.95": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.96": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.97": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.98": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.experts.gate_up_proj_list.99": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.shared_expert.down_proj": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.shared_expert.gate_proj": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 512, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.shared_expert.up_proj": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 512, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.14.mlp.shared_expert_gate": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16, + "out_features": 1, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.0": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.1": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.10": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.100": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.101": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.102": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.103": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.104": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.105": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.106": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.107": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.108": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.109": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.11": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.110": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.111": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.112": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.113": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.114": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.115": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.116": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.117": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.118": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.119": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.12": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.120": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.121": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.122": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.123": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.124": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.125": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.126": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.127": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.128": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.129": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.13": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.130": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.131": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.132": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.133": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.134": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.135": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.136": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.137": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.138": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.139": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.14": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.140": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.141": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.142": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.143": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.144": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.145": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.146": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.147": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.148": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.149": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.15": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.150": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.151": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.152": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.153": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.154": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.155": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.156": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.157": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.158": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.159": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.16": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.160": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.161": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.162": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.163": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.164": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.165": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.166": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.167": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.168": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.169": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.17": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.170": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.171": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.172": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.173": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.174": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.175": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.176": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.177": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.178": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.179": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.18": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.180": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.181": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.182": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.183": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.184": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.185": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.186": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.187": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.188": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.189": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.19": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.190": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.191": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.192": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.193": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.194": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.195": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.196": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.197": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.198": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.199": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.2": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.20": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.200": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.201": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.202": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.203": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.204": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.205": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.206": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.207": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.208": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.209": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.21": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.210": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.211": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.212": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.213": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.214": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.215": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.216": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.217": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.218": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.219": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.22": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.220": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.221": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.222": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.223": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.224": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.225": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.226": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.227": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.228": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.229": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.23": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.230": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.231": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.232": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.233": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.234": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.235": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.236": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.237": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.238": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.239": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.24": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.240": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.241": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.242": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.243": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.244": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.245": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.246": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.247": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.248": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.249": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.25": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.250": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.251": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.252": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.253": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.254": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.255": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.26": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.27": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.28": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.29": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.3": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.30": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.31": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.32": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.33": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.34": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.35": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.36": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.37": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.38": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.39": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.4": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.40": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.41": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.42": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.43": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.44": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.45": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.46": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.47": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.48": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.49": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.5": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.50": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.51": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.52": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.53": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.54": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.55": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.56": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.57": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.58": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.59": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.6": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.60": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.61": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.62": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.63": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.64": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.65": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.66": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.67": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.68": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.69": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.7": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.70": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.71": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.72": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.73": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.74": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.75": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.76": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.77": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.78": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.79": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.8": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.80": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.81": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.82": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.83": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.84": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.85": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.86": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.87": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.88": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.89": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.9": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.90": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.91": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.92": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.93": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.94": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.95": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.96": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.97": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.98": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.down_proj_list.99": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.0": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.1": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.10": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.100": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.101": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.102": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.103": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.104": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.105": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.106": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.107": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.108": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.109": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.11": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.110": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.111": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.112": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.113": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.114": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.115": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.116": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.117": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.118": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.119": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.12": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.120": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.121": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.122": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.123": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.124": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.125": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.126": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.127": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.128": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.129": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.13": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.130": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.131": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.132": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.133": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.134": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.135": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.136": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.137": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.138": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.139": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.14": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.140": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.141": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.142": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.143": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.144": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.145": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.146": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.147": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.148": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.149": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.15": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.150": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.151": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.152": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.153": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.154": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.155": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.156": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.157": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.158": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.159": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.16": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.160": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.161": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.162": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.163": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.164": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.165": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.166": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.167": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.168": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.169": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.17": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.170": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.171": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.172": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.173": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.174": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.175": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.176": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.177": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.178": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.179": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.18": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.180": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.181": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.182": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.183": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.184": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.185": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.186": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.187": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.188": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.189": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.19": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.190": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.191": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.192": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.193": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.194": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.195": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.196": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.197": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.198": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.199": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.2": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.20": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.200": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.201": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.202": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.203": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.204": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.205": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.206": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.207": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.208": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.209": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.21": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.210": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.211": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.212": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.213": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.214": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.215": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.216": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.217": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.218": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.219": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.22": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.220": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.221": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.222": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.223": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.224": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.225": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.226": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.227": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.228": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.229": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.23": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.230": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.231": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.232": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.233": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.234": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.235": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.236": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.237": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.238": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.239": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.24": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.240": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.241": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.242": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.243": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.244": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.245": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.246": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.247": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.248": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.249": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.25": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.250": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.251": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.252": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.253": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.254": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.255": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.26": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.27": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.28": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.29": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.3": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.30": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.31": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.32": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.33": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.34": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.35": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.36": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.37": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.38": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.39": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.4": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.40": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.41": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.42": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.43": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.44": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.45": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.46": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.47": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.48": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.49": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.5": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.50": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.51": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.52": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.53": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.54": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.55": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.56": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.57": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.58": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.59": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.6": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.60": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.61": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.62": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.63": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.64": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.65": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.66": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.67": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.68": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.69": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.7": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.70": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.71": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.72": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.73": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.74": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.75": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.76": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.77": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.78": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.79": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.8": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.80": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.81": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.82": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.83": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.84": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.85": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.86": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.87": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.88": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.89": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.9": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.90": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.91": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.92": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.93": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.94": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.95": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.96": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.97": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.98": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.experts.gate_up_proj_list.99": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.shared_expert.down_proj": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.shared_expert.gate_proj": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 512, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.shared_expert.up_proj": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 512, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.mlp.shared_expert_gate": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16, + "out_features": 1, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.self_attn.k_proj": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 512, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.self_attn.o_proj": { + "group_size": 128, + "in_features": 4096, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 65536, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 4096, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.self_attn.q_proj": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 131072, + "out_features": 8192, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.15.self_attn.v_proj": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 512, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.linear_attn.in_proj_a": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 512, + "out_features": 32, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.linear_attn.in_proj_b": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 512, + "out_features": 32, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.linear_attn.in_proj_qkv": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 131072, + "out_features": 8192, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.linear_attn.in_proj_z": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 65536, + "out_features": 4096, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.linear_attn.out_proj": { + "group_size": 128, + "in_features": 4096, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 65536, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 4096, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.0": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.1": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.10": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.100": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.101": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.102": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.103": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.104": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.105": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.106": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.107": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.108": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.109": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.11": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.110": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.111": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.112": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.113": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.114": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.115": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.116": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.117": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.118": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.119": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.12": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.120": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.121": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.122": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.123": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.124": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.125": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.126": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.127": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.128": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.129": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.13": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.130": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.131": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.132": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.133": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.134": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.135": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.136": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.137": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.138": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.139": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.14": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.140": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.141": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.142": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.143": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.144": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.145": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.146": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.147": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.148": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.149": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.15": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.150": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.151": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.152": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.153": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.154": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.155": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.156": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.157": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.158": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.159": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.16": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.160": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.161": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.162": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.163": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.164": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.165": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.166": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.167": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.168": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.169": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.17": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.170": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.171": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.172": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.173": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.174": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.175": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.176": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.177": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.178": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.179": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.18": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.180": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.181": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.182": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.183": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.184": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.185": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.186": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.187": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.188": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.189": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.19": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.190": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.191": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.192": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.193": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.194": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.195": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.196": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.197": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.198": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.199": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.2": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.20": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.200": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.201": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.202": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.203": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.204": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.205": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.206": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.207": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.208": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.209": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.21": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.210": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.211": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.212": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.213": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.214": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.215": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.216": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.217": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.218": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.219": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.22": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.220": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.221": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.222": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.223": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.224": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.225": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.226": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.227": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.228": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.229": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.23": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.230": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.231": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.232": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.233": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.234": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.235": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.236": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.237": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.238": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.239": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.24": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.240": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.241": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.242": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.243": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.244": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.245": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.246": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.247": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.248": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.249": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.25": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.250": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.251": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.252": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.253": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.254": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.255": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.26": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.27": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.28": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.29": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.3": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.30": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.31": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.32": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.33": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.34": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.35": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.36": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.37": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.38": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.39": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.4": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.40": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.41": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.42": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.43": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.44": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.45": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.46": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.47": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.48": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.49": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.5": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.50": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.51": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.52": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.53": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.54": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.55": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.56": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.57": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.58": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.59": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.6": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.60": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.61": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.62": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.63": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.64": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.65": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.66": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.67": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.68": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.69": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.7": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.70": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.71": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.72": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.73": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.74": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.75": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.76": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.77": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.78": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.79": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.8": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.80": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.81": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.82": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.83": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.84": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.85": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.86": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.87": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.88": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.89": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.9": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.90": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.91": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.92": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.93": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.94": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.95": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.96": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.97": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.98": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.down_proj_list.99": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.0": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.1": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.10": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.100": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.101": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.102": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.103": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.104": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.105": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.106": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.107": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.108": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.109": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.11": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.110": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.111": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.112": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.113": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.114": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.115": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.116": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.117": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.118": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.119": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.12": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.120": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.121": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.122": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.123": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.124": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.125": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.126": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.127": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.128": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.129": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.13": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.130": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.131": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.132": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.133": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.134": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.135": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.136": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.137": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.138": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.139": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.14": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.140": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.141": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.142": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.143": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.144": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.145": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.146": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.147": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.148": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.149": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.15": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.150": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.151": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.152": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.153": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.154": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.155": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.156": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.157": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.158": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.159": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.16": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.160": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.161": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.162": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.163": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.164": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.165": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.166": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.167": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.168": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.169": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.17": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.170": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.171": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.172": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.173": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.174": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.175": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.176": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.177": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.178": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.179": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.18": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.180": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.181": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.182": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.183": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.184": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.185": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.186": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.187": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.188": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.189": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.19": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.190": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.191": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.192": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.193": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.194": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.195": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.196": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.197": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.198": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.199": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.2": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.20": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.200": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.201": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.202": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.203": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.204": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.205": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.206": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.207": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.208": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.209": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.21": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.210": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.211": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.212": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.213": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.214": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.215": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.216": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.217": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.218": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.219": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.22": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.220": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.221": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.222": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.223": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.224": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.225": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.226": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.227": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.228": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.229": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.23": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.230": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.231": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.232": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.233": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.234": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.235": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.236": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.237": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.238": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.239": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.24": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.240": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.241": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.242": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.243": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.244": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.245": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.246": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.247": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.248": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.249": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.25": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.250": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.251": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.252": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.253": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.254": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.255": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.26": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.27": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.28": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.29": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.3": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.30": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.31": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.32": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.33": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.34": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.35": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.36": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.37": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.38": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.39": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.4": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.40": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.41": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.42": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.43": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.44": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.45": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.46": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.47": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.48": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.49": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.5": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.50": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.51": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.52": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.53": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.54": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.55": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.56": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.57": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.58": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.59": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.6": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.60": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.61": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.62": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.63": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.64": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.65": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.66": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.67": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.68": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.69": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.7": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.70": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.71": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.72": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.73": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.74": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.75": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.76": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.77": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.78": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.79": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.8": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.80": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.81": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.82": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.83": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.84": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.85": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.86": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.87": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.88": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.89": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.9": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.90": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.91": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.92": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.93": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.94": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.95": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.96": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.97": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.98": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.experts.gate_up_proj_list.99": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.shared_expert.down_proj": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.shared_expert.gate_proj": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 512, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.shared_expert.up_proj": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 512, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.16.mlp.shared_expert_gate": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16, + "out_features": 1, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.linear_attn.in_proj_a": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 512, + "out_features": 32, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.linear_attn.in_proj_b": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 512, + "out_features": 32, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.linear_attn.in_proj_qkv": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 131072, + "out_features": 8192, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.linear_attn.in_proj_z": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 65536, + "out_features": 4096, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.linear_attn.out_proj": { + "group_size": 128, + "in_features": 4096, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 65536, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 4096, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.0": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.1": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.10": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.100": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.101": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.102": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.103": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.104": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.105": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.106": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.107": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.108": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.109": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.11": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.110": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.111": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.112": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.113": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.114": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.115": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.116": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.117": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.118": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.119": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.12": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.120": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.121": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.122": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.123": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.124": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.125": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.126": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.127": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.128": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.129": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.13": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.130": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.131": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.132": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.133": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.134": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.135": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.136": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.137": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.138": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.139": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.14": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.140": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.141": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.142": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.143": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.144": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.145": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.146": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.147": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.148": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.149": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.15": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.150": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.151": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.152": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.153": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.154": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.155": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.156": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.157": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.158": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.159": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.16": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.160": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.161": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.162": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.163": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.164": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.165": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.166": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.167": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.168": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.169": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.17": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.170": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.171": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.172": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.173": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.174": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.175": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.176": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.177": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.178": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.179": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.18": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.180": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.181": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.182": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.183": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.184": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.185": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.186": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.187": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.188": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.189": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.19": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.190": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.191": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.192": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.193": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.194": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.195": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.196": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.197": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.198": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.199": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.2": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.20": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.200": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.201": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.202": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.203": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.204": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.205": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.206": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.207": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.208": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.209": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.21": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.210": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.211": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.212": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.213": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.214": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.215": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.216": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.217": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.218": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.219": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.22": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.220": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.221": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.222": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.223": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.224": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.225": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.226": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.227": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.228": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.229": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.23": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.230": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.231": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.232": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.233": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.234": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.235": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.236": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.237": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.238": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.239": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.24": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.240": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.241": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.242": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.243": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.244": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.245": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.246": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.247": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.248": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.249": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.25": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.250": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.251": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.252": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.253": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.254": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.255": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.26": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.27": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.28": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.29": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.3": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.30": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.31": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.32": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.33": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.34": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.35": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.36": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.37": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.38": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.39": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.4": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.40": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.41": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.42": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.43": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.44": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.45": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.46": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.47": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.48": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.49": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.5": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.50": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.51": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.52": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.53": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.54": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.55": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.56": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.57": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.58": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.59": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.6": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.60": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.61": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.62": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.63": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.64": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.65": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.66": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.67": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.68": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.69": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.7": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.70": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.71": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.72": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.73": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.74": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.75": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.76": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.77": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.78": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.79": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.8": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.80": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.81": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.82": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.83": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.84": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.85": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.86": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.87": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.88": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.89": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.9": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.90": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.91": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.92": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.93": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.94": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.95": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.96": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.97": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.98": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.down_proj_list.99": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.0": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.1": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.10": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.100": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.101": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.102": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.103": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.104": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.105": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.106": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.107": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.108": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.109": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.11": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.110": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.111": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.112": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.113": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.114": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.115": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.116": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.117": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.118": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.119": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.12": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.120": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.121": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.122": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.123": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.124": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.125": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.126": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.127": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.128": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.129": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.13": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.130": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.131": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.132": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.133": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.134": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.135": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.136": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.137": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.138": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.139": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.14": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.140": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.141": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.142": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.143": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.144": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.145": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.146": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.147": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.148": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.149": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.15": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.150": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.151": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.152": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.153": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.154": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.155": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.156": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.157": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.158": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.159": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.16": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.160": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.161": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.162": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.163": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.164": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.165": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.166": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.167": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.168": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.169": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.17": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.170": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.171": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.172": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.173": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.174": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.175": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.176": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.177": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.178": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.179": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.18": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.180": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.181": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.182": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.183": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.184": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.185": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.186": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.187": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.188": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.189": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.19": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.190": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.191": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.192": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.193": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.194": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.195": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.196": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.197": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.198": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.199": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.2": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.20": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.200": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.201": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.202": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.203": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.204": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.205": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.206": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.207": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.208": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.209": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.21": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.210": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.211": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.212": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.213": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.214": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.215": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.216": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.217": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.218": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.219": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.22": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.220": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.221": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.222": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.223": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.224": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.225": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.226": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.227": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.228": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.229": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.23": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.230": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.231": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.232": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.233": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.234": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.235": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.236": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.237": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.238": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.239": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.24": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.240": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.241": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.242": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.243": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.244": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.245": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.246": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.247": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.248": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.249": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.25": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.250": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.251": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.252": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.253": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.254": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.255": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.26": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.27": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.28": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.29": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.3": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.30": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.31": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.32": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.33": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.34": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.35": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.36": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.37": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.38": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.39": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.4": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.40": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.41": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.42": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.43": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.44": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.45": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.46": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.47": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.48": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.49": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.5": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.50": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.51": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.52": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.53": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.54": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.55": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.56": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.57": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.58": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.59": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.6": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.60": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.61": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.62": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.63": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.64": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.65": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.66": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.67": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.68": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.69": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.7": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.70": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.71": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.72": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.73": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.74": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.75": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.76": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.77": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.78": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.79": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.8": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.80": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.81": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.82": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.83": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.84": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.85": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.86": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.87": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.88": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.89": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.9": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.90": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.91": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.92": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.93": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.94": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.95": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.96": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.97": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.98": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.experts.gate_up_proj_list.99": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.shared_expert.down_proj": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.shared_expert.gate_proj": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 512, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.shared_expert.up_proj": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 512, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.17.mlp.shared_expert_gate": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16, + "out_features": 1, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.linear_attn.in_proj_a": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 512, + "out_features": 32, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.linear_attn.in_proj_b": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 512, + "out_features": 32, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.linear_attn.in_proj_qkv": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 131072, + "out_features": 8192, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.linear_attn.in_proj_z": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 65536, + "out_features": 4096, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.linear_attn.out_proj": { + "group_size": 128, + "in_features": 4096, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 65536, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 4096, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.0": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.1": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.10": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.100": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.101": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.102": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.103": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.104": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.105": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.106": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.107": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.108": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.109": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.11": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.110": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.111": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.112": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.113": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.114": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.115": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.116": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.117": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.118": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.119": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.12": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.120": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.121": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.122": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.123": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.124": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.125": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.126": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.127": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.128": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.129": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.13": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.130": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.131": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.132": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.133": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.134": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.135": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.136": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.137": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.138": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.139": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.14": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.140": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.141": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.142": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.143": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.144": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.145": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.146": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.147": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.148": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.149": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.15": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.150": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.151": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.152": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.153": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.154": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.155": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.156": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.157": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.158": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.159": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.16": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.160": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.161": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.162": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.163": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.164": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.165": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.166": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.167": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.168": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.169": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.17": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.170": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.171": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.172": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.173": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.174": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.175": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.176": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.177": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.178": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.179": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.18": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.180": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.181": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.182": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.183": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.184": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.185": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.186": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.187": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.188": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.189": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.19": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.190": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.191": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.192": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.193": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.194": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.195": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.196": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.197": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.198": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.199": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.2": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.20": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.200": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.201": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.202": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.203": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.204": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.205": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.206": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.207": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.208": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.209": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.21": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.210": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.211": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.212": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.213": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.214": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.215": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.216": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.217": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.218": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.219": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.22": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.220": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.221": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.222": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.223": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.224": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.225": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.226": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.227": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.228": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.229": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.23": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.230": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.231": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.232": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.233": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.234": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.235": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.236": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.237": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.238": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.239": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.24": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.240": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.241": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.242": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.243": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.244": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.245": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.246": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.247": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.248": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.249": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.25": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.250": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.251": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.252": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.253": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.254": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.255": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.26": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.27": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.28": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.29": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.3": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.30": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.31": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.32": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.33": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.34": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.35": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.36": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.37": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.38": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.39": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.4": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.40": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.41": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.42": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.43": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.44": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.45": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.46": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.47": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.48": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.49": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.5": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.50": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.51": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.52": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.53": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.54": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.55": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.56": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.57": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.58": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.59": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.6": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.60": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.61": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.62": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.63": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.64": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.65": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.66": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.67": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.68": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.69": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.7": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.70": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.71": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.72": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.73": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.74": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.75": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.76": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.77": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.78": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.79": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.8": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.80": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.81": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.82": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.83": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.84": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.85": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.86": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.87": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.88": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.89": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.9": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.90": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.91": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.92": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.93": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.94": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.95": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.96": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.97": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.98": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.down_proj_list.99": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.0": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.1": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.10": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.100": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.101": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.102": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.103": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.104": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.105": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.106": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.107": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.108": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.109": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.11": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.110": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.111": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.112": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.113": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.114": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.115": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.116": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.117": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.118": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.119": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.12": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.120": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.121": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.122": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.123": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.124": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.125": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.126": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.127": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.128": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.129": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.13": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.130": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.131": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.132": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.133": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.134": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.135": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.136": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.137": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.138": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.139": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.14": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.140": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.141": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.142": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.143": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.144": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.145": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.146": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.147": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.148": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.149": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.15": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.150": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.151": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.152": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.153": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.154": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.155": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.156": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.157": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.158": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.159": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.16": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.160": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.161": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.162": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.163": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.164": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.165": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.166": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.167": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.168": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.169": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.17": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.170": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.171": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.172": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.173": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.174": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.175": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.176": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.177": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.178": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.179": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.18": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.180": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.181": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.182": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.183": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.184": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.185": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.186": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.187": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.188": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.189": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.19": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.190": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.191": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.192": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.193": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.194": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.195": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.196": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.197": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.198": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.199": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.2": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.20": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.200": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.201": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.202": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.203": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.204": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.205": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.206": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.207": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.208": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.209": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.21": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.210": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.211": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.212": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.213": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.214": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.215": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.216": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.217": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.218": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.219": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.22": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.220": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.221": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.222": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.223": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.224": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.225": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.226": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.227": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.228": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.229": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.23": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.230": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.231": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.232": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.233": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.234": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.235": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.236": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.237": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.238": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.239": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.24": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.240": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.241": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.242": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.243": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.244": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.245": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.246": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.247": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.248": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.249": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.25": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.250": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.251": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.252": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.253": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.254": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.255": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.26": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.27": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.28": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.29": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.3": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.30": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.31": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.32": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.33": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.34": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.35": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.36": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.37": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.38": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.39": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.4": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.40": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.41": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.42": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.43": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.44": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.45": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.46": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.47": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.48": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.49": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.5": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.50": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.51": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.52": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.53": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.54": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.55": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.56": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.57": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.58": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.59": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.6": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.60": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.61": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.62": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.63": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.64": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.65": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.66": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.67": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.68": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.69": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.7": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.70": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.71": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.72": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.73": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.74": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.75": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.76": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.77": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.78": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.79": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.8": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.80": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.81": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.82": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.83": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.84": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.85": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.86": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.87": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.88": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.89": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.9": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.90": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.91": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.92": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.93": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.94": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.95": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.96": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.97": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.98": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.experts.gate_up_proj_list.99": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.shared_expert.down_proj": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.shared_expert.gate_proj": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 512, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.shared_expert.up_proj": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 512, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.18.mlp.shared_expert_gate": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16, + "out_features": 1, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.0": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.1": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.10": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.100": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.101": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.102": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.103": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.104": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.105": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.106": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.107": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.108": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.109": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.11": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.110": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.111": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.112": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.113": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.114": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.115": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.116": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.117": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.118": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.119": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.12": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.120": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.121": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.122": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.123": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.124": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.125": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.126": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.127": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.128": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.129": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.13": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.130": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.131": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.132": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.133": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.134": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.135": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.136": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.137": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.138": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.139": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.14": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.140": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.141": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.142": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.143": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.144": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.145": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.146": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.147": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.148": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.149": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.15": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.150": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.151": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.152": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.153": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.154": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.155": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.156": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.157": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.158": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.159": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.16": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.160": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.161": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.162": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.163": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.164": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.165": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.166": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.167": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.168": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.169": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.17": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.170": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.171": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.172": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.173": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.174": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.175": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.176": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.177": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.178": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.179": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.18": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.180": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.181": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.182": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.183": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.184": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.185": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.186": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.187": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.188": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.189": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.19": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.190": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.191": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.192": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.193": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.194": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.195": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.196": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.197": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.198": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.199": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.2": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.20": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.200": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.201": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.202": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.203": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.204": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.205": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.206": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.207": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.208": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.209": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.21": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.210": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.211": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.212": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.213": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.214": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.215": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.216": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.217": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.218": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.219": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.22": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.220": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.221": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.222": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.223": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.224": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.225": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.226": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.227": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.228": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.229": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.23": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.230": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.231": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.232": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.233": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.234": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.235": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.236": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.237": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.238": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.239": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.24": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.240": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.241": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.242": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.243": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.244": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.245": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.246": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.247": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.248": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.249": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.25": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.250": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.251": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.252": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.253": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.254": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.255": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.26": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.27": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.28": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.29": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.3": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.30": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.31": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.32": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.33": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.34": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.35": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.36": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.37": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.38": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.39": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.4": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.40": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.41": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.42": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.43": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.44": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.45": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.46": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.47": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.48": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.49": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.5": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.50": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.51": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.52": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.53": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.54": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.55": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.56": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.57": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.58": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.59": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.6": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.60": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.61": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.62": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.63": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.64": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.65": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.66": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.67": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.68": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.69": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.7": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.70": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.71": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.72": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.73": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.74": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.75": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.76": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.77": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.78": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.79": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.8": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.80": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.81": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.82": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.83": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.84": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.85": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.86": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.87": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.88": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.89": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.9": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.90": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.91": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.92": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.93": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.94": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.95": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.96": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.97": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.98": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.down_proj_list.99": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.0": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.1": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.10": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.100": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.101": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.102": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.103": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.104": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.105": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.106": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.107": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.108": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.109": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.11": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.110": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.111": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.112": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.113": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.114": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.115": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.116": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.117": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.118": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.119": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.12": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.120": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.121": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.122": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.123": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.124": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.125": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.126": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.127": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.128": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.129": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.13": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.130": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.131": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.132": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.133": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.134": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.135": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.136": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.137": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.138": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.139": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.14": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.140": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.141": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.142": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.143": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.144": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.145": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.146": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.147": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.148": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.149": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.15": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.150": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.151": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.152": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.153": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.154": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.155": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.156": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.157": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.158": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.159": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.16": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.160": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.161": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.162": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.163": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.164": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.165": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.166": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.167": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.168": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.169": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.17": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.170": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.171": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.172": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.173": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.174": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.175": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.176": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.177": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.178": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.179": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.18": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.180": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.181": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.182": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.183": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.184": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.185": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.186": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.187": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.188": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.189": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.19": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.190": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.191": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.192": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.193": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.194": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.195": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.196": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.197": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.198": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.199": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.2": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.20": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.200": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.201": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.202": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.203": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.204": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.205": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.206": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.207": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.208": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.209": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.21": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.210": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.211": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.212": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.213": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.214": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.215": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.216": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.217": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.218": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.219": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.22": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.220": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.221": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.222": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.223": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.224": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.225": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.226": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.227": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.228": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.229": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.23": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.230": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.231": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.232": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.233": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.234": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.235": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.236": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.237": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.238": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.239": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.24": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.240": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.241": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.242": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.243": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.244": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.245": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.246": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.247": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.248": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.249": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.25": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.250": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.251": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.252": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.253": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.254": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.255": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.26": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.27": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.28": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.29": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.3": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.30": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.31": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.32": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.33": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.34": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.35": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.36": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.37": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.38": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.39": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.4": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.40": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.41": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.42": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.43": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.44": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.45": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.46": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.47": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.48": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.49": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.5": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.50": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.51": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.52": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.53": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.54": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.55": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.56": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.57": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.58": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.59": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.6": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.60": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.61": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.62": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.63": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.64": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.65": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.66": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.67": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.68": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.69": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.7": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.70": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.71": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.72": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.73": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.74": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.75": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.76": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.77": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.78": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.79": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.8": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.80": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.81": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.82": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.83": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.84": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.85": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.86": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.87": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.88": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.89": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.9": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.90": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.91": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.92": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.93": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.94": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.95": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.96": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.97": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.98": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.experts.gate_up_proj_list.99": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.shared_expert.down_proj": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.shared_expert.gate_proj": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 512, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.shared_expert.up_proj": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 512, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.mlp.shared_expert_gate": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16, + "out_features": 1, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.self_attn.k_proj": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 512, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.self_attn.o_proj": { + "group_size": 128, + "in_features": 4096, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 65536, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 4096, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.self_attn.q_proj": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 131072, + "out_features": 8192, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.19.self_attn.v_proj": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 512, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.linear_attn.in_proj_a": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 512, + "out_features": 32, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.linear_attn.in_proj_b": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 512, + "out_features": 32, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.linear_attn.in_proj_qkv": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 131072, + "out_features": 8192, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.linear_attn.in_proj_z": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 65536, + "out_features": 4096, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.linear_attn.out_proj": { + "group_size": 128, + "in_features": 4096, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 65536, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 4096, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.0": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.1": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.10": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.100": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.101": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.102": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.103": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.104": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.105": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.106": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.107": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.108": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.109": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.11": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.110": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.111": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.112": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.113": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.114": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.115": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.116": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.117": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.118": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.119": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.12": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.120": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.121": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.122": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.123": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.124": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.125": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.126": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.127": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.128": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.129": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.13": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.130": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.131": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.132": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.133": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.134": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.135": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.136": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.137": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.138": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.139": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.14": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.140": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.141": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.142": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.143": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.144": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.145": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.146": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.147": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.148": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.149": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.15": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.150": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.151": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.152": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.153": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.154": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.155": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.156": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.157": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.158": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.159": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.16": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.160": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.161": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.162": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.163": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.164": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.165": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.166": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.167": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.168": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.169": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.17": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.170": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.171": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.172": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.173": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.174": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.175": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.176": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.177": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.178": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.179": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.18": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.180": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.181": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.182": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.183": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.184": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.185": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.186": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.187": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.188": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.189": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.19": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.190": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.191": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.192": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.193": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.194": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.195": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.196": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.197": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.198": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.199": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.2": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.20": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.200": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.201": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.202": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.203": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.204": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.205": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.206": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.207": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.208": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.209": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.21": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.210": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.211": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.212": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.213": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.214": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.215": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.216": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.217": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.218": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.219": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.22": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.220": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.221": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.222": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.223": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.224": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.225": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.226": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.227": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.228": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.229": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.23": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.230": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.231": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.232": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.233": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.234": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.235": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.236": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.237": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.238": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.239": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.24": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.240": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.241": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.242": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.243": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.244": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.245": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.246": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.247": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.248": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.249": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.25": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.250": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.251": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.252": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.253": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.254": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.255": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.26": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.27": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.28": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.29": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.3": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.30": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.31": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.32": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.33": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.34": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.35": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.36": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.37": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.38": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.39": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.4": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.40": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.41": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.42": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.43": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.44": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.45": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.46": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.47": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.48": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.49": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.5": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.50": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.51": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.52": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.53": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.54": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.55": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.56": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.57": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.58": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.59": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.6": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.60": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.61": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.62": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.63": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.64": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.65": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.66": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.67": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.68": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.69": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.7": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.70": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.71": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.72": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.73": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.74": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.75": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.76": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.77": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.78": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.79": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.8": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.80": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.81": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.82": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.83": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.84": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.85": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.86": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.87": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.88": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.89": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.9": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.90": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.91": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.92": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.93": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.94": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.95": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.96": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.97": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.98": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.down_proj_list.99": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.0": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.1": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.10": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.100": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.101": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.102": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.103": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.104": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.105": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.106": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.107": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.108": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.109": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.11": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.110": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.111": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.112": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.113": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.114": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.115": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.116": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.117": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.118": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.119": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.12": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.120": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.121": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.122": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.123": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.124": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.125": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.126": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.127": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.128": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.129": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.13": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.130": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.131": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.132": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.133": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.134": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.135": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.136": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.137": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.138": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.139": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.14": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.140": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.141": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.142": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.143": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.144": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.145": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.146": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.147": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.148": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.149": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.15": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.150": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.151": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.152": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.153": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.154": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.155": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.156": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.157": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.158": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.159": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.16": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.160": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.161": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.162": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.163": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.164": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.165": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.166": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.167": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.168": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.169": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.17": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.170": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.171": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.172": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.173": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.174": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.175": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.176": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.177": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.178": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.179": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.18": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.180": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.181": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.182": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.183": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.184": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.185": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.186": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.187": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.188": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.189": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.19": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.190": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.191": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.192": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.193": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.194": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.195": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.196": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.197": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.198": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.199": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.2": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.20": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.200": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.201": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.202": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.203": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.204": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.205": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.206": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.207": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.208": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.209": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.21": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.210": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.211": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.212": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.213": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.214": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.215": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.216": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.217": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.218": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.219": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.22": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.220": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.221": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.222": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.223": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.224": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.225": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.226": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.227": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.228": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.229": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.23": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.230": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.231": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.232": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.233": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.234": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.235": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.236": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.237": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.238": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.239": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.24": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.240": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.241": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.242": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.243": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.244": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.245": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.246": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.247": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.248": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.249": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.25": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.250": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.251": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.252": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.253": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.254": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.255": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.26": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.27": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.28": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.29": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.3": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.30": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.31": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.32": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.33": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.34": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.35": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.36": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.37": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.38": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.39": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.4": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.40": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.41": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.42": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.43": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.44": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.45": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.46": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.47": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.48": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.49": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.5": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.50": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.51": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.52": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.53": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.54": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.55": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.56": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.57": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.58": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.59": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.6": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.60": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.61": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.62": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.63": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.64": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.65": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.66": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.67": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.68": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.69": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.7": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.70": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.71": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.72": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.73": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.74": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.75": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.76": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.77": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.78": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.79": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.8": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.80": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.81": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.82": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.83": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.84": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.85": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.86": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.87": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.88": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.89": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.9": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.90": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.91": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.92": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.93": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.94": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.95": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.96": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.97": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.98": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.experts.gate_up_proj_list.99": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.shared_expert.down_proj": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.shared_expert.gate_proj": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 512, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.shared_expert.up_proj": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 512, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.2.mlp.shared_expert_gate": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16, + "out_features": 1, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.linear_attn.in_proj_a": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 512, + "out_features": 32, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.linear_attn.in_proj_b": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 512, + "out_features": 32, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.linear_attn.in_proj_qkv": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 131072, + "out_features": 8192, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.linear_attn.in_proj_z": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 65536, + "out_features": 4096, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.linear_attn.out_proj": { + "group_size": 128, + "in_features": 4096, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 65536, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 4096, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.0": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.1": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.10": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.100": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.101": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.102": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.103": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.104": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.105": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.106": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.107": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.108": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.109": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.11": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.110": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.111": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.112": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.113": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.114": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.115": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.116": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.117": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.118": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.119": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.12": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.120": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.121": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.122": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.123": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.124": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.125": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.126": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.127": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.128": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.129": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.13": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.130": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.131": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.132": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.133": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.134": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.135": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.136": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.137": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.138": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.139": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.14": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.140": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.141": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.142": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.143": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.144": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.145": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.146": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.147": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.148": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.149": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.15": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.150": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.151": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.152": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.153": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.154": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.155": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.156": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.157": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.158": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.159": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.16": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.160": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.161": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.162": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.163": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.164": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.165": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.166": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.167": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.168": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.169": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.17": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.170": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.171": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.172": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.173": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.174": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.175": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.176": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.177": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.178": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.179": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.18": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.180": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.181": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.182": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.183": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.184": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.185": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.186": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.187": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.188": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.189": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.19": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.190": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.191": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.192": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.193": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.194": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.195": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.196": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.197": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.198": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.199": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.2": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.20": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.200": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.201": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.202": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.203": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.204": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.205": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.206": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.207": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.208": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.209": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.21": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.210": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.211": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.212": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.213": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.214": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.215": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.216": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.217": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.218": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.219": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.22": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.220": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.221": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.222": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.223": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.224": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.225": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.226": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.227": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.228": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.229": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.23": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.230": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.231": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.232": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.233": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.234": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.235": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.236": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.237": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.238": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.239": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.24": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.240": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.241": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.242": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.243": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.244": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.245": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.246": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.247": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.248": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.249": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.25": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.250": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.251": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.252": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.253": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.254": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.255": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.26": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.27": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.28": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.29": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.3": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.30": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.31": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.32": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.33": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.34": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.35": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.36": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.37": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.38": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.39": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.4": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.40": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.41": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.42": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.43": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.44": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.45": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.46": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.47": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.48": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.49": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.5": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.50": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.51": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.52": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.53": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.54": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.55": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.56": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.57": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.58": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.59": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.6": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.60": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.61": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.62": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.63": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.64": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.65": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.66": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.67": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.68": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.69": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.7": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.70": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.71": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.72": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.73": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.74": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.75": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.76": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.77": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.78": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.79": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.8": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.80": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.81": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.82": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.83": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.84": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.85": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.86": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.87": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.88": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.89": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.9": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.90": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.91": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.92": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.93": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.94": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.95": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.96": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.97": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.98": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.down_proj_list.99": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.0": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.1": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.10": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.100": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.101": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.102": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.103": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.104": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.105": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.106": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.107": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.108": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.109": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.11": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.110": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.111": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.112": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.113": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.114": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.115": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.116": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.117": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.118": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.119": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.12": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.120": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.121": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.122": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.123": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.124": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.125": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.126": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.127": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.128": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.129": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.13": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.130": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.131": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.132": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.133": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.134": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.135": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.136": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.137": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.138": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.139": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.14": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.140": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.141": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.142": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.143": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.144": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.145": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.146": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.147": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.148": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.149": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.15": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.150": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.151": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.152": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.153": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.154": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.155": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.156": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.157": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.158": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.159": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.16": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.160": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.161": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.162": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.163": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.164": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.165": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.166": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.167": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.168": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.169": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.17": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.170": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.171": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.172": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.173": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.174": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.175": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.176": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.177": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.178": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.179": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.18": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.180": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.181": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.182": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.183": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.184": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.185": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.186": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.187": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.188": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.189": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.19": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.190": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.191": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.192": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.193": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.194": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.195": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.196": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.197": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.198": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.199": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.2": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.20": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.200": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.201": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.202": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.203": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.204": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.205": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.206": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.207": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.208": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.209": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.21": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.210": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.211": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.212": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.213": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.214": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.215": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.216": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.217": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.218": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.219": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.22": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.220": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.221": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.222": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.223": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.224": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.225": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.226": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.227": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.228": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.229": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.23": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.230": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.231": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.232": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.233": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.234": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.235": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.236": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.237": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.238": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.239": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.24": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.240": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.241": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.242": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.243": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.244": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.245": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.246": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.247": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.248": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.249": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.25": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.250": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.251": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.252": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.253": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.254": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.255": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.26": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.27": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.28": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.29": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.3": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.30": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.31": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.32": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.33": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.34": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.35": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.36": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.37": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.38": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.39": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.4": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.40": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.41": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.42": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.43": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.44": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.45": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.46": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.47": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.48": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.49": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.5": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.50": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.51": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.52": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.53": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.54": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.55": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.56": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.57": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.58": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.59": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.6": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.60": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.61": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.62": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.63": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.64": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.65": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.66": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.67": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.68": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.69": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.7": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.70": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.71": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.72": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.73": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.74": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.75": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.76": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.77": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.78": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.79": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.8": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.80": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.81": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.82": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.83": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.84": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.85": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.86": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.87": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.88": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.89": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.9": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.90": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.91": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.92": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.93": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.94": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.95": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.96": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.97": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.98": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.experts.gate_up_proj_list.99": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.shared_expert.down_proj": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.shared_expert.gate_proj": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 512, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.shared_expert.up_proj": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 512, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.20.mlp.shared_expert_gate": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16, + "out_features": 1, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.linear_attn.in_proj_a": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 512, + "out_features": 32, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.linear_attn.in_proj_b": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 512, + "out_features": 32, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.linear_attn.in_proj_qkv": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 131072, + "out_features": 8192, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.linear_attn.in_proj_z": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 65536, + "out_features": 4096, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.linear_attn.out_proj": { + "group_size": 128, + "in_features": 4096, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 65536, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 4096, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.0": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.1": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.10": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.100": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.101": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.102": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.103": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.104": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.105": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.106": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.107": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.108": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.109": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.11": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.110": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.111": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.112": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.113": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.114": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.115": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.116": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.117": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.118": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.119": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.12": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.120": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.121": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.122": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.123": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.124": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.125": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.126": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.127": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.128": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.129": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.13": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.130": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.131": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.132": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.133": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.134": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.135": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.136": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.137": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.138": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.139": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.14": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.140": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.141": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.142": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.143": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.144": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.145": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.146": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.147": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.148": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.149": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.15": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.150": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.151": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.152": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.153": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.154": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.155": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.156": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.157": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.158": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.159": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.16": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.160": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.161": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.162": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.163": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.164": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.165": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.166": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.167": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.168": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.169": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.17": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.170": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.171": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.172": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.173": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.174": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.175": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.176": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.177": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.178": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.179": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.18": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.180": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.181": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.182": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.183": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.184": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.185": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.186": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.187": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.188": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.189": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.19": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.190": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.191": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.192": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.193": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.194": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.195": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.196": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.197": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.198": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.199": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.2": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.20": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.200": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.201": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.202": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.203": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.204": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.205": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.206": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.207": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.208": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.209": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.21": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.210": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.211": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.212": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.213": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.214": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.215": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.216": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.217": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.218": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.219": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.22": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.220": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.221": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.222": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.223": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.224": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.225": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.226": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.227": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.228": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.229": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.23": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.230": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.231": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.232": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.233": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.234": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.235": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.236": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.237": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.238": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.239": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.24": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.240": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.241": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.242": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.243": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.244": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.245": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.246": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.247": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.248": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.249": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.25": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.250": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.251": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.252": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.253": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.254": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.255": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.26": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.27": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.28": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.29": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.3": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.30": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.31": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.32": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.33": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.34": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.35": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.36": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.37": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.38": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.39": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.4": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.40": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.41": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.42": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.43": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.44": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.45": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.46": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.47": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.48": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.49": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.5": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.50": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.51": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.52": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.53": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.54": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.55": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.56": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.57": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.58": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.59": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.6": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.60": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.61": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.62": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.63": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.64": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.65": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.66": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.67": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.68": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.69": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.7": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.70": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.71": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.72": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.73": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.74": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.75": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.76": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.77": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.78": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.79": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.8": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.80": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.81": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.82": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.83": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.84": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.85": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.86": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.87": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.88": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.89": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.9": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.90": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.91": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.92": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.93": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.94": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.95": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.96": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.97": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.98": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.down_proj_list.99": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.0": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.1": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.10": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.100": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.101": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.102": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.103": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.104": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.105": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.106": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.107": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.108": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.109": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.11": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.110": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.111": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.112": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.113": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.114": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.115": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.116": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.117": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.118": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.119": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.12": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.120": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.121": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.122": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.123": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.124": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.125": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.126": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.127": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.128": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.129": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.13": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.130": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.131": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.132": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.133": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.134": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.135": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.136": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.137": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.138": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.139": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.14": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.140": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.141": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.142": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.143": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.144": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.145": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.146": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.147": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.148": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.149": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.15": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.150": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.151": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.152": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.153": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.154": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.155": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.156": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.157": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.158": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.159": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.16": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.160": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.161": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.162": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.163": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.164": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.165": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.166": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.167": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.168": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.169": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.17": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.170": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.171": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.172": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.173": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.174": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.175": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.176": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.177": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.178": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.179": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.18": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.180": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.181": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.182": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.183": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.184": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.185": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.186": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.187": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.188": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.189": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.19": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.190": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.191": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.192": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.193": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.194": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.195": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.196": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.197": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.198": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.199": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.2": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.20": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.200": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.201": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.202": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.203": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.204": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.205": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.206": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.207": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.208": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.209": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.21": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.210": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.211": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.212": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.213": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.214": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.215": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.216": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.217": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.218": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.219": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.22": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.220": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.221": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.222": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.223": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.224": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.225": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.226": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.227": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.228": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.229": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.23": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.230": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.231": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.232": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.233": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.234": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.235": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.236": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.237": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.238": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.239": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.24": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.240": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.241": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.242": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.243": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.244": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.245": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.246": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.247": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.248": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.249": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.25": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.250": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.251": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.252": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.253": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.254": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.255": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.26": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.27": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.28": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.29": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.3": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.30": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.31": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.32": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.33": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.34": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.35": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.36": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.37": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.38": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.39": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.4": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.40": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.41": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.42": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.43": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.44": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.45": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.46": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.47": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.48": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.49": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.5": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.50": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.51": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.52": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.53": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.54": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.55": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.56": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.57": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.58": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.59": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.6": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.60": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.61": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.62": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.63": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.64": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.65": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.66": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.67": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.68": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.69": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.7": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.70": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.71": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.72": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.73": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.74": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.75": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.76": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.77": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.78": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.79": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.8": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.80": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.81": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.82": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.83": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.84": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.85": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.86": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.87": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.88": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.89": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.9": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.90": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.91": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.92": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.93": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.94": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.95": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.96": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.97": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.98": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.experts.gate_up_proj_list.99": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.shared_expert.down_proj": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.shared_expert.gate_proj": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 512, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.shared_expert.up_proj": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 512, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.21.mlp.shared_expert_gate": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16, + "out_features": 1, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.linear_attn.in_proj_a": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 512, + "out_features": 32, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.linear_attn.in_proj_b": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 512, + "out_features": 32, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.linear_attn.in_proj_qkv": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 131072, + "out_features": 8192, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.linear_attn.in_proj_z": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 65536, + "out_features": 4096, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.linear_attn.out_proj": { + "group_size": 128, + "in_features": 4096, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 65536, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 4096, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.0": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.1": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.10": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.100": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.101": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.102": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.103": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.104": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.105": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.106": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.107": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.108": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.109": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.11": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.110": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.111": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.112": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.113": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.114": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.115": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.116": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.117": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.118": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.119": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.12": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.120": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.121": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.122": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.123": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.124": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.125": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.126": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.127": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.128": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.129": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.13": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.130": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.131": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.132": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.133": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.134": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.135": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.136": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.137": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.138": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.139": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.14": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.140": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.141": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.142": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.143": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.144": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.145": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.146": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.147": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.148": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.149": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.15": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.150": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.151": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.152": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.153": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.154": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.155": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.156": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.157": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.158": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.159": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.16": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.160": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.161": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.162": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.163": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.164": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.165": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.166": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.167": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.168": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.169": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.17": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.170": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.171": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.172": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.173": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.174": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.175": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.176": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.177": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.178": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.179": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.18": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.180": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.181": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.182": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.183": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.184": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.185": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.186": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.187": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.188": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.189": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.19": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.190": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.191": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.192": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.193": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.194": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.195": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.196": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.197": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.198": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.199": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.2": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.20": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.200": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.201": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.202": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.203": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.204": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.205": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.206": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.207": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.208": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.209": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.21": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.210": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.211": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.212": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.213": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.214": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.215": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.216": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.217": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.218": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.219": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.22": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.220": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.221": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.222": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.223": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.224": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.225": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.226": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.227": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.228": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.229": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.23": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.230": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.231": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.232": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.233": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.234": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.235": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.236": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.237": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.238": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.239": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.24": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.240": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.241": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.242": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.243": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.244": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.245": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.246": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.247": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.248": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.249": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.25": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.250": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.251": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.252": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.253": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.254": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.255": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.26": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.27": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.28": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.29": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.3": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.30": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.31": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.32": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.33": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.34": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.35": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.36": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.37": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.38": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.39": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.4": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.40": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.41": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.42": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.43": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.44": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.45": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.46": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.47": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.48": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.49": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.5": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.50": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.51": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.52": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.53": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.54": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.55": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.56": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.57": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.58": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.59": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.6": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.60": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.61": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.62": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.63": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.64": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.65": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.66": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.67": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.68": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.69": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.7": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.70": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.71": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.72": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.73": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.74": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.75": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.76": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.77": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.78": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.79": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.8": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.80": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.81": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.82": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.83": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.84": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.85": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.86": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.87": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.88": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.89": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.9": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.90": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.91": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.92": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.93": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.94": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.95": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.96": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.97": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.98": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.down_proj_list.99": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.0": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.1": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.10": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.100": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.101": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.102": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.103": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.104": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.105": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.106": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.107": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.108": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.109": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.11": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.110": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.111": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.112": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.113": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.114": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.115": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.116": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.117": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.118": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.119": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.12": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.120": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.121": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.122": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.123": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.124": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.125": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.126": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.127": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.128": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.129": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.13": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.130": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.131": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.132": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.133": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.134": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.135": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.136": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.137": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.138": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.139": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.14": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.140": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.141": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.142": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.143": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.144": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.145": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.146": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.147": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.148": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.149": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.15": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.150": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.151": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.152": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.153": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.154": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.155": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.156": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.157": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.158": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.159": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.16": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.160": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.161": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.162": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.163": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.164": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.165": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.166": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.167": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.168": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.169": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.17": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.170": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.171": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.172": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.173": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.174": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.175": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.176": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.177": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.178": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.179": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.18": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.180": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.181": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.182": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.183": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.184": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.185": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.186": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.187": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.188": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.189": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.19": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.190": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.191": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.192": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.193": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.194": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.195": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.196": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.197": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.198": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.199": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.2": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.20": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.200": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.201": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.202": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.203": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.204": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.205": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.206": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.207": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.208": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.209": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.21": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.210": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.211": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.212": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.213": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.214": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.215": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.216": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.217": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.218": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.219": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.22": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.220": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.221": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.222": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.223": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.224": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.225": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.226": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.227": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.228": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.229": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.23": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.230": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.231": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.232": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.233": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.234": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.235": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.236": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.237": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.238": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.239": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.24": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.240": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.241": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.242": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.243": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.244": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.245": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.246": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.247": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.248": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.249": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.25": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.250": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.251": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.252": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.253": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.254": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.255": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.26": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.27": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.28": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.29": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.3": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.30": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.31": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.32": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.33": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.34": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.35": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.36": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.37": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.38": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.39": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.4": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.40": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.41": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.42": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.43": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.44": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.45": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.46": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.47": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.48": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.49": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.5": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.50": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.51": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.52": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.53": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.54": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.55": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.56": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.57": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.58": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.59": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.6": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.60": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.61": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.62": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.63": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.64": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.65": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.66": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.67": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.68": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.69": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.7": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.70": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.71": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.72": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.73": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.74": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.75": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.76": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.77": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.78": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.79": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.8": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.80": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.81": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.82": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.83": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.84": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.85": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.86": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.87": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.88": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.89": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.9": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.90": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.91": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.92": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.93": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.94": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.95": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.96": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.97": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.98": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.experts.gate_up_proj_list.99": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.shared_expert.down_proj": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.shared_expert.gate_proj": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 512, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.shared_expert.up_proj": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 512, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.22.mlp.shared_expert_gate": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16, + "out_features": 1, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.0": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.1": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.10": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.100": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.101": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.102": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.103": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.104": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.105": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.106": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.107": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.108": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.109": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.11": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.110": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.111": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.112": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.113": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.114": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.115": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.116": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.117": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.118": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.119": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.12": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.120": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.121": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.122": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.123": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.124": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.125": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.126": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.127": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.128": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.129": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.13": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.130": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.131": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.132": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.133": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.134": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.135": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.136": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.137": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.138": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.139": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.14": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.140": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.141": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.142": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.143": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.144": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.145": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.146": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.147": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.148": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.149": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.15": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.150": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.151": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.152": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.153": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.154": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.155": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.156": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.157": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.158": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.159": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.16": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.160": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.161": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.162": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.163": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.164": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.165": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.166": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.167": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.168": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.169": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.17": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.170": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.171": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.172": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.173": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.174": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.175": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.176": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.177": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.178": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.179": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.18": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.180": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.181": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.182": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.183": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.184": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.185": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.186": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.187": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.188": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.189": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.19": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.190": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.191": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.192": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.193": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.194": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.195": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.196": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.197": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.198": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.199": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.2": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.20": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.200": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.201": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.202": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.203": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.204": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.205": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.206": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.207": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.208": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.209": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.21": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.210": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.211": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.212": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.213": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.214": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.215": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.216": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.217": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.218": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.219": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.22": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.220": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.221": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.222": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.223": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.224": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.225": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.226": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.227": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.228": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.229": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.23": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.230": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.231": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.232": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.233": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.234": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.235": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.236": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.237": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.238": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.239": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.24": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.240": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.241": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.242": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.243": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.244": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.245": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.246": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.247": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.248": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.249": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.25": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.250": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.251": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.252": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.253": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.254": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.255": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.26": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.27": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.28": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.29": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.3": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.30": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.31": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.32": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.33": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.34": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.35": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.36": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.37": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.38": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.39": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.4": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.40": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.41": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.42": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.43": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.44": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.45": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.46": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.47": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.48": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.49": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.5": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.50": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.51": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.52": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.53": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.54": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.55": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.56": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.57": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.58": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.59": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.6": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.60": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.61": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.62": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.63": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.64": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.65": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.66": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.67": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.68": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.69": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.7": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.70": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.71": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.72": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.73": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.74": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.75": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.76": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.77": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.78": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.79": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.8": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.80": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.81": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.82": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.83": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.84": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.85": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.86": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.87": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.88": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.89": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.9": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.90": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.91": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.92": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.93": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.94": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.95": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.96": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.97": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.98": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.down_proj_list.99": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.0": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.1": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.10": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.100": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.101": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.102": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.103": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.104": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.105": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.106": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.107": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.108": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.109": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.11": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.110": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.111": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.112": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.113": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.114": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.115": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.116": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.117": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.118": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.119": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.12": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.120": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.121": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.122": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.123": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.124": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.125": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.126": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.127": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.128": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.129": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.13": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.130": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.131": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.132": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.133": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.134": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.135": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.136": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.137": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.138": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.139": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.14": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.140": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.141": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.142": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.143": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.144": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.145": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.146": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.147": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.148": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.149": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.15": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.150": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.151": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.152": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.153": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.154": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.155": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.156": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.157": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.158": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.159": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.16": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.160": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.161": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.162": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.163": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.164": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.165": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.166": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.167": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.168": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.169": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.17": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.170": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.171": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.172": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.173": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.174": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.175": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.176": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.177": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.178": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.179": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.18": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.180": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.181": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.182": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.183": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.184": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.185": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.186": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.187": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.188": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.189": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.19": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.190": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.191": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.192": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.193": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.194": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.195": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.196": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.197": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.198": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.199": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.2": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.20": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.200": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.201": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.202": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.203": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.204": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.205": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.206": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.207": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.208": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.209": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.21": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.210": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.211": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.212": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.213": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.214": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.215": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.216": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.217": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.218": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.219": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.22": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.220": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.221": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.222": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.223": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.224": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.225": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.226": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.227": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.228": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.229": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.23": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.230": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.231": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.232": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.233": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.234": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.235": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.236": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.237": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.238": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.239": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.24": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.240": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.241": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.242": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.243": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.244": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.245": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.246": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.247": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.248": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.249": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.25": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.250": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.251": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.252": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.253": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.254": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.255": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.26": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.27": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.28": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.29": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.3": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.30": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.31": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.32": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.33": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.34": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.35": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.36": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.37": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.38": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.39": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.4": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.40": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.41": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.42": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.43": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.44": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.45": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.46": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.47": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.48": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.49": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.5": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.50": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.51": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.52": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.53": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.54": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.55": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.56": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.57": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.58": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.59": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.6": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.60": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.61": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.62": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.63": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.64": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.65": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.66": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.67": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.68": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.69": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.7": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.70": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.71": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.72": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.73": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.74": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.75": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.76": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.77": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.78": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.79": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.8": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.80": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.81": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.82": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.83": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.84": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.85": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.86": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.87": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.88": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.89": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.9": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.90": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.91": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.92": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.93": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.94": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.95": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.96": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.97": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.98": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.experts.gate_up_proj_list.99": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.shared_expert.down_proj": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.shared_expert.gate_proj": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 512, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.shared_expert.up_proj": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 512, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.mlp.shared_expert_gate": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16, + "out_features": 1, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.self_attn.k_proj": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 512, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.self_attn.o_proj": { + "group_size": 128, + "in_features": 4096, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 65536, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 4096, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.self_attn.q_proj": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 131072, + "out_features": 8192, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.23.self_attn.v_proj": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 512, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.linear_attn.in_proj_a": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 512, + "out_features": 32, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.linear_attn.in_proj_b": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 512, + "out_features": 32, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.linear_attn.in_proj_qkv": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 131072, + "out_features": 8192, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.linear_attn.in_proj_z": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 65536, + "out_features": 4096, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.linear_attn.out_proj": { + "group_size": 128, + "in_features": 4096, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 65536, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 4096, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.0": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.1": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.10": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.100": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.101": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.102": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.103": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.104": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.105": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.106": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.107": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.108": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.109": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.11": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.110": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.111": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.112": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.113": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.114": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.115": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.116": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.117": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.118": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.119": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.12": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.120": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.121": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.122": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.123": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.124": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.125": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.126": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.127": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.128": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.129": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.13": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.130": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.131": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.132": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.133": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.134": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.135": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.136": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.137": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.138": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.139": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.14": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.140": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.141": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.142": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.143": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.144": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.145": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.146": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.147": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.148": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.149": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.15": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.150": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.151": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.152": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.153": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.154": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.155": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.156": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.157": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.158": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.159": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.16": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.160": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.161": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.162": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.163": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.164": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.165": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.166": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.167": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.168": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.169": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.17": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.170": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.171": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.172": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.173": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.174": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.175": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.176": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.177": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.178": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.179": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.18": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.180": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.181": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.182": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.183": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.184": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.185": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.186": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.187": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.188": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.189": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.19": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.190": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.191": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.192": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.193": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.194": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.195": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.196": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.197": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.198": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.199": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.2": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.20": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.200": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.201": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.202": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.203": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.204": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.205": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.206": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.207": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.208": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.209": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.21": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.210": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.211": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.212": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.213": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.214": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.215": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.216": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.217": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.218": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.219": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.22": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.220": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.221": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.222": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.223": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.224": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.225": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.226": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.227": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.228": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.229": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.23": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.230": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.231": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.232": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.233": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.234": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.235": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.236": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.237": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.238": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.239": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.24": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.240": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.241": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.242": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.243": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.244": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.245": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.246": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.247": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.248": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.249": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.25": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.250": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.251": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.252": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.253": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.254": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.255": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.26": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.27": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.28": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.29": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.3": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.30": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.31": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.32": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.33": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.34": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.35": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.36": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.37": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.38": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.39": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.4": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.40": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.41": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.42": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.43": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.44": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.45": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.46": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.47": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.48": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.49": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.5": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.50": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.51": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.52": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.53": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.54": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.55": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.56": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.57": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.58": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.59": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.6": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.60": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.61": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.62": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.63": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.64": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.65": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.66": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.67": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.68": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.69": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.7": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.70": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.71": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.72": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.73": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.74": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.75": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.76": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.77": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.78": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.79": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.8": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.80": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.81": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.82": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.83": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.84": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.85": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.86": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.87": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.88": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.89": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.9": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.90": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.91": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.92": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.93": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.94": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.95": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.96": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.97": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.98": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.down_proj_list.99": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.0": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.1": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.10": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.100": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.101": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.102": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.103": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.104": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.105": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.106": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.107": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.108": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.109": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.11": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.110": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.111": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.112": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.113": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.114": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.115": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.116": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.117": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.118": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.119": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.12": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.120": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.121": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.122": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.123": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.124": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.125": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.126": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.127": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.128": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.129": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.13": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.130": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.131": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.132": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.133": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.134": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.135": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.136": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.137": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.138": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.139": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.14": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.140": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.141": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.142": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.143": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.144": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.145": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.146": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.147": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.148": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.149": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.15": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.150": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.151": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.152": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.153": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.154": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.155": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.156": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.157": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.158": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.159": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.16": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.160": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.161": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.162": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.163": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.164": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.165": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.166": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.167": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.168": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.169": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.17": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.170": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.171": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.172": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.173": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.174": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.175": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.176": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.177": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.178": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.179": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.18": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.180": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.181": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.182": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.183": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.184": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.185": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.186": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.187": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.188": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.189": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.19": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.190": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.191": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.192": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.193": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.194": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.195": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.196": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.197": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.198": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.199": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.2": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.20": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.200": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.201": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.202": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.203": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.204": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.205": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.206": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.207": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.208": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.209": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.21": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.210": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.211": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.212": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.213": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.214": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.215": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.216": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.217": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.218": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.219": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.22": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.220": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.221": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.222": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.223": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.224": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.225": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.226": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.227": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.228": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.229": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.23": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.230": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.231": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.232": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.233": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.234": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.235": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.236": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.237": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.238": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.239": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.24": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.240": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.241": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.242": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.243": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.244": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.245": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.246": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.247": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.248": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.249": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.25": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.250": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.251": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.252": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.253": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.254": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.255": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.26": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.27": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.28": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.29": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.3": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.30": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.31": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.32": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.33": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.34": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.35": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.36": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.37": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.38": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.39": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.4": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.40": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.41": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.42": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.43": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.44": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.45": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.46": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.47": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.48": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.49": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.5": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.50": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.51": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.52": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.53": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.54": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.55": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.56": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.57": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.58": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.59": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.6": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.60": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.61": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.62": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.63": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.64": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.65": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.66": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.67": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.68": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.69": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.7": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.70": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.71": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.72": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.73": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.74": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.75": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.76": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.77": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.78": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.79": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.8": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.80": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.81": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.82": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.83": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.84": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.85": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.86": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.87": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.88": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.89": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.9": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.90": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.91": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.92": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.93": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.94": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.95": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.96": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.97": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.98": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.experts.gate_up_proj_list.99": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.shared_expert.down_proj": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.shared_expert.gate_proj": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 512, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.shared_expert.up_proj": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 512, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.24.mlp.shared_expert_gate": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16, + "out_features": 1, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.linear_attn.in_proj_a": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 512, + "out_features": 32, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.linear_attn.in_proj_b": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 512, + "out_features": 32, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.linear_attn.in_proj_qkv": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 131072, + "out_features": 8192, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.linear_attn.in_proj_z": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 65536, + "out_features": 4096, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.linear_attn.out_proj": { + "group_size": 128, + "in_features": 4096, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 65536, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 4096, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.0": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.1": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.10": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.100": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.101": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.102": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.103": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.104": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.105": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.106": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.107": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.108": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.109": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.11": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.110": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.111": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.112": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.113": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.114": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.115": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.116": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.117": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.118": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.119": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.12": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.120": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.121": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.122": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.123": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.124": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.125": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.126": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.127": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.128": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.129": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.13": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.130": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.131": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.132": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.133": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.134": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.135": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.136": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.137": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.138": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.139": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.14": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.140": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.141": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.142": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.143": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.144": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.145": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.146": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.147": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.148": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.149": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.15": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.150": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.151": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.152": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.153": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.154": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.155": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.156": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.157": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.158": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.159": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.16": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.160": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.161": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.162": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.163": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.164": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.165": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.166": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.167": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.168": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.169": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.17": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.170": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.171": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.172": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.173": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.174": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.175": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.176": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.177": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.178": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.179": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.18": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.180": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.181": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.182": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.183": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.184": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.185": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.186": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.187": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.188": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.189": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.19": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.190": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.191": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.192": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.193": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.194": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.195": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.196": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.197": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.198": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.199": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.2": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.20": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.200": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.201": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.202": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.203": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.204": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.205": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.206": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.207": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.208": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.209": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.21": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.210": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.211": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.212": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.213": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.214": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.215": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.216": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.217": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.218": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.219": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.22": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.220": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.221": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.222": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.223": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.224": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.225": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.226": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.227": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.228": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.229": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.23": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.230": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.231": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.232": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.233": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.234": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.235": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.236": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.237": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.238": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.239": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.24": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.240": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.241": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.242": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.243": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.244": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.245": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.246": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.247": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.248": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.249": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.25": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.250": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.251": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.252": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.253": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.254": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.255": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.26": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.27": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.28": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.29": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.3": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.30": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.31": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.32": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.33": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.34": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.35": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.36": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.37": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.38": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.39": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.4": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.40": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.41": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.42": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.43": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.44": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.45": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.46": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.47": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.48": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.49": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.5": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.50": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.51": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.52": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.53": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.54": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.55": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.56": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.57": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.58": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.59": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.6": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.60": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.61": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.62": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.63": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.64": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.65": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.66": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.67": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.68": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.69": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.7": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.70": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.71": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.72": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.73": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.74": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.75": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.76": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.77": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.78": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.79": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.8": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.80": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.81": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.82": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.83": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.84": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.85": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.86": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.87": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.88": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.89": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.9": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.90": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.91": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.92": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.93": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.94": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.95": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.96": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.97": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.98": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.down_proj_list.99": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.0": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.1": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.10": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.100": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.101": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.102": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.103": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.104": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.105": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.106": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.107": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.108": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.109": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.11": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.110": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.111": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.112": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.113": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.114": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.115": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.116": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.117": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.118": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.119": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.12": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.120": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.121": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.122": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.123": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.124": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.125": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.126": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.127": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.128": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.129": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.13": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.130": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.131": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.132": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.133": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.134": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.135": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.136": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.137": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.138": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.139": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.14": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.140": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.141": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.142": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.143": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.144": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.145": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.146": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.147": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.148": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.149": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.15": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.150": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.151": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.152": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.153": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.154": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.155": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.156": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.157": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.158": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.159": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.16": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.160": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.161": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.162": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.163": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.164": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.165": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.166": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.167": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.168": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.169": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.17": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.170": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.171": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.172": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.173": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.174": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.175": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.176": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.177": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.178": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.179": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.18": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.180": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.181": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.182": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.183": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.184": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.185": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.186": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.187": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.188": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.189": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.19": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.190": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.191": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.192": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.193": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.194": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.195": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.196": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.197": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.198": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.199": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.2": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.20": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.200": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.201": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.202": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.203": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.204": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.205": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.206": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.207": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.208": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.209": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.21": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.210": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.211": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.212": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.213": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.214": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.215": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.216": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.217": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.218": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.219": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.22": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.220": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.221": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.222": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.223": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.224": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.225": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.226": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.227": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.228": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.229": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.23": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.230": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.231": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.232": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.233": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.234": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.235": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.236": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.237": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.238": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.239": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.24": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.240": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.241": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.242": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.243": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.244": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.245": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.246": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.247": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.248": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.249": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.25": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.250": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.251": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.252": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.253": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.254": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.255": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.26": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.27": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.28": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.29": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.3": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.30": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.31": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.32": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.33": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.34": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.35": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.36": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.37": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.38": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.39": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.4": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.40": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.41": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.42": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.43": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.44": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.45": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.46": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.47": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.48": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.49": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.5": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.50": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.51": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.52": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.53": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.54": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.55": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.56": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.57": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.58": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.59": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.6": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.60": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.61": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.62": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.63": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.64": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.65": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.66": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.67": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.68": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.69": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.7": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.70": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.71": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.72": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.73": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.74": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.75": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.76": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.77": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.78": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.79": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.8": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.80": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.81": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.82": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.83": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.84": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.85": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.86": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.87": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.88": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.89": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.9": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.90": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.91": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.92": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.93": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.94": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.95": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.96": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.97": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.98": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.experts.gate_up_proj_list.99": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.shared_expert.down_proj": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.shared_expert.gate_proj": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 512, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.shared_expert.up_proj": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 512, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.25.mlp.shared_expert_gate": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16, + "out_features": 1, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.linear_attn.in_proj_a": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 512, + "out_features": 32, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.linear_attn.in_proj_b": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 512, + "out_features": 32, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.linear_attn.in_proj_qkv": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 131072, + "out_features": 8192, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.linear_attn.in_proj_z": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 65536, + "out_features": 4096, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.linear_attn.out_proj": { + "group_size": 128, + "in_features": 4096, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 65536, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 4096, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.0": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.1": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.10": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.100": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.101": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.102": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.103": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.104": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.105": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.106": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.107": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.108": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.109": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.11": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.110": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.111": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.112": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.113": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.114": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.115": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.116": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.117": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.118": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.119": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.12": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.120": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.121": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.122": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.123": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.124": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.125": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.126": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.127": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.128": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.129": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.13": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.130": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.131": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.132": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.133": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.134": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.135": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.136": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.137": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.138": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.139": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.14": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.140": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.141": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.142": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.143": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.144": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.145": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.146": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.147": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.148": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.149": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.15": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.150": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.151": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.152": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.153": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.154": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.155": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.156": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.157": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.158": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.159": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.16": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.160": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.161": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.162": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.163": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.164": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.165": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.166": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.167": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.168": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.169": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.17": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.170": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.171": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.172": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.173": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.174": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.175": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.176": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.177": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.178": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.179": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.18": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.180": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.181": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.182": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.183": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.184": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.185": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.186": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.187": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.188": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.189": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.19": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.190": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.191": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.192": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.193": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.194": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.195": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.196": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.197": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.198": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.199": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.2": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.20": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.200": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.201": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.202": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.203": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.204": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.205": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.206": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.207": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.208": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.209": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.21": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.210": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.211": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.212": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.213": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.214": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.215": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.216": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.217": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.218": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.219": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.22": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.220": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.221": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.222": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.223": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.224": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.225": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.226": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.227": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.228": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.229": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.23": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.230": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.231": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.232": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.233": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.234": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.235": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.236": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.237": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.238": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.239": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.24": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.240": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.241": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.242": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.243": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.244": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.245": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.246": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.247": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.248": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.249": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.25": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.250": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.251": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.252": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.253": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.254": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.255": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.26": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.27": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.28": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.29": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.3": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.30": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.31": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.32": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.33": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.34": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.35": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.36": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.37": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.38": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.39": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.4": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.40": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.41": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.42": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.43": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.44": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.45": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.46": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.47": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.48": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.49": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.5": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.50": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.51": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.52": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.53": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.54": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.55": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.56": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.57": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.58": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.59": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.6": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.60": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.61": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.62": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.63": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.64": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.65": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.66": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.67": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.68": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.69": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.7": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.70": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.71": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.72": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.73": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.74": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.75": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.76": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.77": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.78": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.79": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.8": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.80": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.81": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.82": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.83": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.84": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.85": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.86": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.87": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.88": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.89": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.9": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.90": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.91": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.92": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.93": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.94": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.95": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.96": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.97": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.98": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.down_proj_list.99": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.0": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.1": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.10": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.100": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.101": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.102": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.103": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.104": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.105": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.106": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.107": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.108": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.109": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.11": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.110": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.111": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.112": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.113": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.114": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.115": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.116": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.117": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.118": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.119": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.12": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.120": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.121": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.122": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.123": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.124": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.125": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.126": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.127": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.128": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.129": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.13": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.130": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.131": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.132": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.133": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.134": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.135": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.136": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.137": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.138": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.139": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.14": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.140": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.141": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.142": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.143": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.144": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.145": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.146": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.147": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.148": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.149": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.15": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.150": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.151": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.152": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.153": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.154": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.155": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.156": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.157": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.158": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.159": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.16": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.160": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.161": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.162": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.163": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.164": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.165": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.166": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.167": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.168": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.169": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.17": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.170": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.171": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.172": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.173": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.174": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.175": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.176": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.177": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.178": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.179": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.18": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.180": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.181": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.182": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.183": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.184": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.185": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.186": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.187": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.188": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.189": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.19": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.190": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.191": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.192": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.193": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.194": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.195": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.196": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.197": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.198": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.199": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.2": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.20": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.200": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.201": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.202": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.203": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.204": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.205": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.206": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.207": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.208": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.209": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.21": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.210": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.211": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.212": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.213": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.214": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.215": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.216": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.217": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.218": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.219": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.22": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.220": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.221": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.222": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.223": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.224": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.225": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.226": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.227": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.228": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.229": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.23": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.230": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.231": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.232": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.233": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.234": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.235": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.236": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.237": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.238": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.239": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.24": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.240": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.241": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.242": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.243": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.244": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.245": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.246": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.247": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.248": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.249": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.25": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.250": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.251": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.252": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.253": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.254": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.255": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.26": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.27": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.28": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.29": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.3": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.30": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.31": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.32": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.33": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.34": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.35": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.36": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.37": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.38": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.39": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.4": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.40": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.41": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.42": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.43": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.44": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.45": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.46": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.47": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.48": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.49": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.5": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.50": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.51": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.52": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.53": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.54": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.55": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.56": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.57": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.58": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.59": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.6": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.60": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.61": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.62": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.63": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.64": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.65": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.66": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.67": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.68": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.69": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.7": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.70": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.71": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.72": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.73": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.74": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.75": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.76": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.77": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.78": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.79": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.8": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.80": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.81": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.82": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.83": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.84": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.85": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.86": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.87": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.88": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.89": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.9": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.90": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.91": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.92": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.93": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.94": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.95": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.96": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.97": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.98": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.experts.gate_up_proj_list.99": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.shared_expert.down_proj": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.shared_expert.gate_proj": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 512, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.shared_expert.up_proj": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 512, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.26.mlp.shared_expert_gate": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16, + "out_features": 1, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.0": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.1": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.10": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.100": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.101": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.102": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.103": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.104": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.105": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.106": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.107": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.108": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.109": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.11": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.110": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.111": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.112": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.113": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.114": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.115": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.116": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.117": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.118": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.119": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.12": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.120": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.121": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.122": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.123": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.124": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.125": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.126": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.127": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.128": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.129": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.13": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.130": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.131": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.132": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.133": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.134": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.135": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.136": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.137": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.138": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.139": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.14": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.140": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.141": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.142": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.143": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.144": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.145": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.146": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.147": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.148": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.149": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.15": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.150": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.151": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.152": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.153": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.154": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.155": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.156": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.157": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.158": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.159": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.16": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.160": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.161": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.162": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.163": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.164": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.165": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.166": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.167": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.168": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.169": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.17": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.170": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.171": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.172": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.173": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.174": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.175": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.176": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.177": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.178": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.179": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.18": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.180": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.181": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.182": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.183": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.184": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.185": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.186": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.187": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.188": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.189": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.19": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.190": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.191": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.192": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.193": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.194": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.195": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.196": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.197": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.198": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.199": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.2": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.20": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.200": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.201": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.202": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.203": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.204": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.205": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.206": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.207": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.208": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.209": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.21": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.210": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.211": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.212": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.213": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.214": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.215": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.216": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.217": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.218": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.219": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.22": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.220": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.221": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.222": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.223": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.224": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.225": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.226": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.227": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.228": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.229": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.23": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.230": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.231": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.232": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.233": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.234": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.235": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.236": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.237": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.238": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.239": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.24": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.240": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.241": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.242": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.243": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.244": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.245": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.246": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.247": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.248": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.249": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.25": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.250": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.251": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.252": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.253": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.254": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.255": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.26": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.27": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.28": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.29": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.3": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.30": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.31": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.32": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.33": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.34": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.35": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.36": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.37": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.38": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.39": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.4": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.40": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.41": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.42": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.43": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.44": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.45": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.46": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.47": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.48": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.49": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.5": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.50": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.51": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.52": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.53": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.54": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.55": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.56": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.57": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.58": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.59": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.6": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.60": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.61": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.62": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.63": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.64": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.65": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.66": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.67": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.68": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.69": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.7": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.70": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.71": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.72": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.73": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.74": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.75": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.76": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.77": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.78": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.79": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.8": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.80": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.81": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.82": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.83": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.84": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.85": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.86": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.87": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.88": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.89": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.9": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.90": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.91": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.92": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.93": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.94": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.95": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.96": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.97": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.98": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.down_proj_list.99": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.0": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.1": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.10": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.100": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.101": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.102": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.103": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.104": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.105": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.106": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.107": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.108": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.109": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.11": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.110": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.111": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.112": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.113": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.114": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.115": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.116": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.117": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.118": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.119": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.12": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.120": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.121": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.122": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.123": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.124": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.125": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.126": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.127": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.128": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.129": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.13": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.130": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.131": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.132": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.133": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.134": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.135": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.136": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.137": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.138": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.139": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.14": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.140": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.141": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.142": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.143": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.144": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.145": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.146": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.147": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.148": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.149": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.15": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.150": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.151": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.152": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.153": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.154": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.155": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.156": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.157": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.158": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.159": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.16": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.160": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.161": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.162": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.163": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.164": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.165": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.166": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.167": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.168": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.169": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.17": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.170": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.171": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.172": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.173": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.174": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.175": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.176": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.177": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.178": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.179": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.18": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.180": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.181": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.182": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.183": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.184": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.185": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.186": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.187": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.188": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.189": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.19": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.190": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.191": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.192": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.193": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.194": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.195": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.196": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.197": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.198": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.199": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.2": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.20": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.200": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.201": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.202": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.203": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.204": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.205": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.206": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.207": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.208": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.209": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.21": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.210": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.211": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.212": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.213": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.214": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.215": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.216": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.217": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.218": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.219": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.22": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.220": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.221": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.222": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.223": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.224": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.225": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.226": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.227": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.228": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.229": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.23": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.230": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.231": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.232": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.233": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.234": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.235": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.236": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.237": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.238": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.239": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.24": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.240": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.241": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.242": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.243": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.244": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.245": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.246": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.247": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.248": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.249": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.25": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.250": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.251": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.252": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.253": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.254": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.255": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.26": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.27": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.28": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.29": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.3": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.30": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.31": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.32": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.33": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.34": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.35": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.36": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.37": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.38": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.39": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.4": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.40": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.41": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.42": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.43": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.44": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.45": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.46": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.47": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.48": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.49": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.5": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.50": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.51": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.52": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.53": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.54": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.55": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.56": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.57": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.58": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.59": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.6": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.60": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.61": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.62": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.63": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.64": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.65": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.66": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.67": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.68": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.69": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.7": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.70": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.71": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.72": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.73": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.74": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.75": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.76": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.77": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.78": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.79": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.8": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.80": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.81": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.82": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.83": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.84": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.85": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.86": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.87": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.88": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.89": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.9": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.90": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.91": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.92": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.93": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.94": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.95": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.96": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.97": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.98": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.experts.gate_up_proj_list.99": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.shared_expert.down_proj": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.shared_expert.gate_proj": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 512, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.shared_expert.up_proj": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 512, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.mlp.shared_expert_gate": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16, + "out_features": 1, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.self_attn.k_proj": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 512, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.self_attn.o_proj": { + "group_size": 128, + "in_features": 4096, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 65536, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 4096, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.self_attn.q_proj": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 131072, + "out_features": 8192, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.27.self_attn.v_proj": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 512, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.linear_attn.in_proj_a": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 512, + "out_features": 32, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.linear_attn.in_proj_b": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 512, + "out_features": 32, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.linear_attn.in_proj_qkv": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 131072, + "out_features": 8192, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.linear_attn.in_proj_z": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 65536, + "out_features": 4096, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.linear_attn.out_proj": { + "group_size": 128, + "in_features": 4096, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 65536, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 4096, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.0": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.1": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.10": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.100": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.101": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.102": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.103": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.104": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.105": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.106": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.107": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.108": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.109": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.11": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.110": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.111": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.112": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.113": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.114": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.115": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.116": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.117": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.118": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.119": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.12": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.120": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.121": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.122": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.123": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.124": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.125": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.126": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.127": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.128": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.129": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.13": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.130": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.131": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.132": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.133": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.134": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.135": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.136": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.137": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.138": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.139": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.14": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.140": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.141": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.142": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.143": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.144": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.145": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.146": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.147": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.148": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.149": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.15": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.150": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.151": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.152": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.153": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.154": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.155": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.156": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.157": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.158": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.159": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.16": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.160": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.161": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.162": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.163": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.164": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.165": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.166": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.167": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.168": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.169": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.17": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.170": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.171": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.172": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.173": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.174": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.175": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.176": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.177": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.178": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.179": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.18": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.180": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.181": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.182": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.183": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.184": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.185": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.186": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.187": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.188": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.189": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.19": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.190": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.191": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.192": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.193": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.194": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.195": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.196": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.197": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.198": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.199": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.2": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.20": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.200": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.201": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.202": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.203": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.204": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.205": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.206": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.207": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.208": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.209": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.21": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.210": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.211": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.212": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.213": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.214": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.215": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.216": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.217": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.218": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.219": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.22": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.220": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.221": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.222": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.223": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.224": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.225": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.226": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.227": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.228": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.229": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.23": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.230": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.231": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.232": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.233": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.234": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.235": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.236": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.237": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.238": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.239": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.24": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.240": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.241": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.242": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.243": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.244": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.245": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.246": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.247": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.248": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.249": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.25": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.250": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.251": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.252": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.253": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.254": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.255": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.26": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.27": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.28": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.29": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.3": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.30": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.31": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.32": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.33": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.34": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.35": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.36": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.37": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.38": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.39": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.4": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.40": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.41": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.42": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.43": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.44": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.45": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.46": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.47": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.48": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.49": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.5": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.50": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.51": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.52": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.53": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.54": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.55": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.56": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.57": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.58": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.59": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.6": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.60": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.61": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.62": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.63": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.64": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.65": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.66": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.67": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.68": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.69": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.7": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.70": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.71": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.72": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.73": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.74": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.75": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.76": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.77": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.78": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.79": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.8": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.80": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.81": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.82": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.83": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.84": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.85": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.86": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.87": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.88": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.89": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.9": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.90": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.91": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.92": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.93": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.94": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.95": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.96": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.97": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.98": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.down_proj_list.99": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.0": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.1": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.10": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.100": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.101": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.102": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.103": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.104": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.105": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.106": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.107": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.108": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.109": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.11": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.110": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.111": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.112": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.113": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.114": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.115": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.116": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.117": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.118": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.119": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.12": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.120": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.121": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.122": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.123": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.124": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.125": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.126": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.127": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.128": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.129": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.13": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.130": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.131": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.132": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.133": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.134": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.135": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.136": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.137": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.138": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.139": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.14": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.140": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.141": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.142": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.143": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.144": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.145": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.146": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.147": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.148": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.149": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.15": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.150": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.151": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.152": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.153": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.154": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.155": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.156": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.157": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.158": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.159": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.16": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.160": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.161": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.162": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.163": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.164": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.165": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.166": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.167": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.168": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.169": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.17": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.170": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.171": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.172": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.173": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.174": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.175": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.176": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.177": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.178": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.179": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.18": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.180": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.181": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.182": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.183": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.184": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.185": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.186": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.187": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.188": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.189": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.19": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.190": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.191": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.192": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.193": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.194": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.195": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.196": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.197": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.198": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.199": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.2": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.20": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.200": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.201": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.202": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.203": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.204": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.205": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.206": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.207": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.208": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.209": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.21": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.210": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.211": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.212": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.213": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.214": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.215": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.216": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.217": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.218": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.219": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.22": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.220": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.221": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.222": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.223": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.224": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.225": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.226": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.227": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.228": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.229": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.23": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.230": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.231": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.232": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.233": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.234": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.235": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.236": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.237": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.238": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.239": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.24": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.240": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.241": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.242": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.243": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.244": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.245": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.246": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.247": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.248": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.249": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.25": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.250": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.251": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.252": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.253": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.254": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.255": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.26": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.27": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.28": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.29": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.3": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.30": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.31": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.32": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.33": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.34": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.35": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.36": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.37": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.38": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.39": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.4": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.40": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.41": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.42": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.43": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.44": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.45": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.46": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.47": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.48": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.49": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.5": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.50": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.51": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.52": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.53": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.54": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.55": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.56": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.57": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.58": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.59": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.6": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.60": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.61": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.62": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.63": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.64": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.65": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.66": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.67": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.68": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.69": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.7": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.70": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.71": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.72": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.73": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.74": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.75": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.76": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.77": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.78": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.79": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.8": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.80": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.81": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.82": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.83": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.84": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.85": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.86": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.87": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.88": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.89": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.9": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.90": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.91": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.92": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.93": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.94": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.95": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.96": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.97": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.98": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.experts.gate_up_proj_list.99": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.shared_expert.down_proj": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.shared_expert.gate_proj": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 512, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.shared_expert.up_proj": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 512, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.28.mlp.shared_expert_gate": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16, + "out_features": 1, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.linear_attn.in_proj_a": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 512, + "out_features": 32, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.linear_attn.in_proj_b": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 512, + "out_features": 32, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.linear_attn.in_proj_qkv": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 131072, + "out_features": 8192, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.linear_attn.in_proj_z": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 65536, + "out_features": 4096, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.linear_attn.out_proj": { + "group_size": 128, + "in_features": 4096, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 65536, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 4096, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.0": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.1": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.10": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.100": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.101": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.102": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.103": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.104": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.105": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.106": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.107": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.108": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.109": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.11": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.110": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.111": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.112": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.113": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.114": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.115": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.116": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.117": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.118": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.119": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.12": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.120": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.121": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.122": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.123": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.124": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.125": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.126": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.127": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.128": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.129": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.13": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.130": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.131": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.132": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.133": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.134": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.135": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.136": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.137": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.138": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.139": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.14": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.140": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.141": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.142": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.143": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.144": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.145": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.146": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.147": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.148": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.149": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.15": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.150": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.151": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.152": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.153": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.154": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.155": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.156": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.157": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.158": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.159": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.16": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.160": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.161": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.162": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.163": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.164": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.165": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.166": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.167": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.168": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.169": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.17": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.170": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.171": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.172": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.173": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.174": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.175": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.176": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.177": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.178": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.179": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.18": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.180": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.181": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.182": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.183": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.184": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.185": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.186": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.187": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.188": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.189": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.19": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.190": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.191": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.192": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.193": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.194": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.195": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.196": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.197": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.198": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.199": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.2": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.20": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.200": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.201": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.202": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.203": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.204": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.205": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.206": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.207": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.208": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.209": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.21": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.210": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.211": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.212": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.213": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.214": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.215": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.216": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.217": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.218": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.219": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.22": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.220": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.221": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.222": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.223": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.224": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.225": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.226": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.227": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.228": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.229": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.23": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.230": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.231": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.232": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.233": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.234": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.235": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.236": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.237": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.238": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.239": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.24": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.240": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.241": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.242": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.243": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.244": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.245": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.246": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.247": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.248": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.249": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.25": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.250": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.251": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.252": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.253": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.254": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.255": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.26": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.27": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.28": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.29": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.3": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.30": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.31": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.32": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.33": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.34": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.35": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.36": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.37": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.38": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.39": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.4": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.40": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.41": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.42": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.43": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.44": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.45": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.46": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.47": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.48": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.49": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.5": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.50": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.51": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.52": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.53": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.54": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.55": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.56": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.57": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.58": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.59": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.6": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.60": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.61": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.62": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.63": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.64": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.65": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.66": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.67": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.68": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.69": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.7": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.70": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.71": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.72": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.73": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.74": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.75": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.76": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.77": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.78": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.79": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.8": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.80": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.81": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.82": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.83": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.84": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.85": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.86": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.87": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.88": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.89": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.9": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.90": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.91": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.92": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.93": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.94": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.95": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.96": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.97": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.98": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.down_proj_list.99": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.0": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.1": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.10": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.100": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.101": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.102": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.103": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.104": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.105": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.106": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.107": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.108": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.109": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.11": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.110": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.111": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.112": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.113": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.114": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.115": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.116": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.117": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.118": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.119": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.12": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.120": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.121": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.122": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.123": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.124": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.125": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.126": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.127": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.128": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.129": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.13": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.130": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.131": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.132": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.133": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.134": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.135": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.136": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.137": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.138": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.139": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.14": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.140": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.141": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.142": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.143": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.144": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.145": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.146": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.147": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.148": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.149": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.15": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.150": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.151": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.152": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.153": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.154": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.155": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.156": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.157": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.158": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.159": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.16": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.160": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.161": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.162": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.163": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.164": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.165": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.166": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.167": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.168": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.169": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.17": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.170": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.171": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.172": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.173": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.174": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.175": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.176": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.177": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.178": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.179": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.18": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.180": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.181": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.182": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.183": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.184": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.185": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.186": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.187": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.188": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.189": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.19": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.190": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.191": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.192": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.193": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.194": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.195": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.196": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.197": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.198": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.199": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.2": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.20": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.200": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.201": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.202": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.203": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.204": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.205": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.206": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.207": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.208": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.209": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.21": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.210": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.211": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.212": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.213": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.214": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.215": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.216": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.217": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.218": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.219": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.22": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.220": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.221": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.222": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.223": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.224": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.225": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.226": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.227": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.228": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.229": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.23": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.230": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.231": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.232": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.233": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.234": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.235": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.236": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.237": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.238": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.239": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.24": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.240": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.241": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.242": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.243": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.244": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.245": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.246": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.247": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.248": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.249": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.25": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.250": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.251": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.252": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.253": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.254": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.255": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.26": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.27": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.28": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.29": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.3": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.30": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.31": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.32": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.33": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.34": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.35": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.36": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.37": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.38": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.39": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.4": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.40": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.41": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.42": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.43": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.44": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.45": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.46": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.47": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.48": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.49": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.5": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.50": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.51": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.52": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.53": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.54": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.55": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.56": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.57": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.58": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.59": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.6": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.60": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.61": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.62": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.63": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.64": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.65": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.66": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.67": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.68": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.69": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.7": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.70": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.71": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.72": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.73": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.74": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.75": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.76": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.77": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.78": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.79": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.8": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.80": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.81": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.82": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.83": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.84": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.85": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.86": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.87": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.88": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.89": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.9": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.90": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.91": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.92": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.93": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.94": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.95": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.96": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.97": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.98": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.experts.gate_up_proj_list.99": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.shared_expert.down_proj": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.shared_expert.gate_proj": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 512, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.shared_expert.up_proj": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 512, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.29.mlp.shared_expert_gate": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16, + "out_features": 1, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.0": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.1": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.10": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.100": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.101": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.102": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.103": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.104": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.105": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.106": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.107": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.108": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.109": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.11": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.110": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.111": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.112": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.113": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.114": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.115": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.116": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.117": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.118": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.119": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.12": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.120": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.121": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.122": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.123": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.124": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.125": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.126": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.127": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.128": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.129": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.13": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.130": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.131": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.132": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.133": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.134": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.135": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.136": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.137": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.138": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.139": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.14": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.140": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.141": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.142": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.143": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.144": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.145": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.146": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.147": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.148": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.149": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.15": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.150": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.151": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.152": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.153": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.154": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.155": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.156": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.157": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.158": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.159": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.16": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.160": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.161": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.162": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.163": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.164": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.165": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.166": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.167": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.168": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.169": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.17": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.170": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.171": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.172": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.173": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.174": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.175": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.176": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.177": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.178": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.179": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.18": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.180": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.181": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.182": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.183": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.184": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.185": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.186": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.187": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.188": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.189": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.19": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.190": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.191": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.192": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.193": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.194": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.195": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.196": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.197": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.198": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.199": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.2": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.20": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.200": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.201": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.202": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.203": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.204": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.205": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.206": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.207": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.208": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.209": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.21": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.210": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.211": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.212": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.213": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.214": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.215": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.216": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.217": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.218": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.219": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.22": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.220": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.221": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.222": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.223": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.224": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.225": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.226": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.227": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.228": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.229": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.23": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.230": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.231": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.232": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.233": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.234": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.235": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.236": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.237": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.238": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.239": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.24": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.240": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.241": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.242": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.243": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.244": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.245": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.246": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.247": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.248": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.249": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.25": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.250": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.251": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.252": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.253": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.254": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.255": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.26": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.27": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.28": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.29": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.3": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.30": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.31": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.32": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.33": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.34": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.35": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.36": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.37": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.38": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.39": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.4": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.40": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.41": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.42": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.43": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.44": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.45": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.46": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.47": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.48": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.49": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.5": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.50": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.51": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.52": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.53": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.54": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.55": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.56": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.57": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.58": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.59": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.6": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.60": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.61": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.62": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.63": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.64": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.65": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.66": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.67": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.68": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.69": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.7": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.70": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.71": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.72": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.73": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.74": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.75": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.76": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.77": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.78": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.79": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.8": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.80": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.81": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.82": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.83": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.84": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.85": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.86": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.87": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.88": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.89": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.9": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.90": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.91": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.92": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.93": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.94": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.95": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.96": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.97": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.98": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.down_proj_list.99": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.0": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.1": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.10": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.100": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.101": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.102": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.103": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.104": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.105": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.106": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.107": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.108": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.109": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.11": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.110": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.111": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.112": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.113": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.114": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.115": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.116": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.117": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.118": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.119": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.12": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.120": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.121": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.122": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.123": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.124": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.125": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.126": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.127": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.128": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.129": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.13": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.130": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.131": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.132": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.133": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.134": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.135": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.136": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.137": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.138": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.139": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.14": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.140": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.141": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.142": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.143": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.144": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.145": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.146": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.147": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.148": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.149": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.15": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.150": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.151": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.152": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.153": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.154": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.155": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.156": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.157": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.158": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.159": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.16": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.160": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.161": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.162": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.163": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.164": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.165": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.166": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.167": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.168": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.169": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.17": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.170": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.171": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.172": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.173": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.174": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.175": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.176": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.177": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.178": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.179": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.18": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.180": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.181": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.182": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.183": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.184": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.185": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.186": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.187": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.188": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.189": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.19": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.190": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.191": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.192": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.193": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.194": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.195": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.196": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.197": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.198": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.199": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.2": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.20": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.200": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.201": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.202": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.203": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.204": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.205": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.206": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.207": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.208": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.209": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.21": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.210": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.211": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.212": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.213": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.214": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.215": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.216": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.217": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.218": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.219": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.22": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.220": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.221": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.222": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.223": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.224": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.225": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.226": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.227": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.228": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.229": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.23": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.230": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.231": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.232": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.233": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.234": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.235": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.236": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.237": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.238": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.239": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.24": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.240": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.241": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.242": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.243": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.244": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.245": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.246": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.247": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.248": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.249": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.25": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.250": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.251": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.252": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.253": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.254": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.255": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.26": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.27": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.28": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.29": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.3": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.30": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.31": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.32": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.33": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.34": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.35": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.36": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.37": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.38": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.39": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.4": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.40": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.41": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.42": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.43": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.44": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.45": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.46": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.47": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.48": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.49": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.5": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.50": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.51": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.52": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.53": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.54": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.55": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.56": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.57": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.58": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.59": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.6": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.60": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.61": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.62": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.63": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.64": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.65": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.66": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.67": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.68": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.69": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.7": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.70": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.71": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.72": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.73": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.74": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.75": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.76": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.77": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.78": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.79": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.8": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.80": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.81": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.82": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.83": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.84": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.85": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.86": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.87": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.88": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.89": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.9": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.90": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.91": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.92": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.93": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.94": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.95": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.96": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.97": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.98": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.experts.gate_up_proj_list.99": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.shared_expert.down_proj": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.shared_expert.gate_proj": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 512, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.shared_expert.up_proj": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 512, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.mlp.shared_expert_gate": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16, + "out_features": 1, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.self_attn.k_proj": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 512, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.self_attn.o_proj": { + "group_size": 128, + "in_features": 4096, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 65536, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 4096, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.self_attn.q_proj": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 131072, + "out_features": 8192, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.3.self_attn.v_proj": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 512, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.linear_attn.in_proj_a": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 512, + "out_features": 32, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.linear_attn.in_proj_b": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 512, + "out_features": 32, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.linear_attn.in_proj_qkv": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 131072, + "out_features": 8192, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.linear_attn.in_proj_z": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 65536, + "out_features": 4096, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.linear_attn.out_proj": { + "group_size": 128, + "in_features": 4096, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 65536, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 4096, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.0": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.1": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.10": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.100": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.101": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.102": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.103": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.104": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.105": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.106": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.107": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.108": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.109": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.11": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.110": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.111": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.112": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.113": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.114": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.115": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.116": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.117": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.118": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.119": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.12": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.120": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.121": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.122": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.123": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.124": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.125": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.126": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.127": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.128": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.129": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.13": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.130": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.131": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.132": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.133": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.134": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.135": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.136": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.137": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.138": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.139": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.14": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.140": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.141": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.142": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.143": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.144": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.145": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.146": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.147": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.148": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.149": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.15": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.150": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.151": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.152": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.153": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.154": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.155": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.156": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.157": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.158": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.159": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.16": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.160": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.161": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.162": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.163": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.164": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.165": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.166": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.167": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.168": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.169": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.17": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.170": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.171": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.172": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.173": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.174": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.175": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.176": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.177": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.178": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.179": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.18": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.180": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.181": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.182": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.183": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.184": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.185": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.186": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.187": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.188": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.189": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.19": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.190": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.191": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.192": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.193": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.194": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.195": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.196": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.197": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.198": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.199": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.2": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.20": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.200": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.201": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.202": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.203": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.204": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.205": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.206": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.207": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.208": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.209": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.21": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.210": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.211": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.212": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.213": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.214": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.215": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.216": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.217": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.218": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.219": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.22": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.220": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.221": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.222": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.223": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.224": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.225": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.226": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.227": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.228": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.229": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.23": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.230": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.231": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.232": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.233": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.234": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.235": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.236": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.237": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.238": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.239": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.24": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.240": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.241": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.242": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.243": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.244": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.245": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.246": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.247": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.248": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.249": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.25": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.250": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.251": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.252": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.253": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.254": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.255": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.26": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.27": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.28": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.29": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.3": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.30": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.31": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.32": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.33": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.34": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.35": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.36": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.37": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.38": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.39": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.4": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.40": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.41": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.42": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.43": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.44": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.45": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.46": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.47": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.48": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.49": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.5": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.50": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.51": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.52": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.53": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.54": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.55": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.56": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.57": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.58": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.59": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.6": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.60": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.61": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.62": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.63": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.64": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.65": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.66": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.67": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.68": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.69": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.7": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.70": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.71": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.72": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.73": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.74": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.75": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.76": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.77": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.78": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.79": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.8": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.80": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.81": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.82": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.83": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.84": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.85": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.86": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.87": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.88": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.89": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.9": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.90": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.91": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.92": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.93": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.94": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.95": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.96": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.97": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.98": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.down_proj_list.99": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.0": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.1": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.10": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.100": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.101": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.102": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.103": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.104": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.105": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.106": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.107": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.108": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.109": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.11": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.110": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.111": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.112": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.113": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.114": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.115": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.116": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.117": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.118": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.119": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.12": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.120": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.121": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.122": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.123": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.124": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.125": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.126": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.127": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.128": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.129": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.13": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.130": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.131": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.132": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.133": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.134": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.135": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.136": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.137": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.138": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.139": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.14": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.140": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.141": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.142": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.143": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.144": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.145": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.146": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.147": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.148": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.149": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.15": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.150": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.151": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.152": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.153": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.154": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.155": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.156": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.157": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.158": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.159": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.16": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.160": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.161": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.162": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.163": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.164": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.165": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.166": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.167": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.168": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.169": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.17": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.170": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.171": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.172": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.173": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.174": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.175": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.176": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.177": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.178": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.179": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.18": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.180": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.181": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.182": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.183": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.184": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.185": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.186": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.187": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.188": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.189": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.19": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.190": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.191": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.192": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.193": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.194": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.195": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.196": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.197": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.198": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.199": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.2": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.20": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.200": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.201": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.202": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.203": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.204": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.205": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.206": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.207": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.208": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.209": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.21": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.210": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.211": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.212": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.213": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.214": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.215": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.216": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.217": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.218": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.219": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.22": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.220": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.221": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.222": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.223": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.224": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.225": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.226": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.227": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.228": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.229": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.23": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.230": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.231": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.232": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.233": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.234": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.235": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.236": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.237": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.238": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.239": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.24": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.240": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.241": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.242": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.243": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.244": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.245": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.246": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.247": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.248": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.249": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.25": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.250": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.251": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.252": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.253": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.254": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.255": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.26": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.27": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.28": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.29": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.3": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.30": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.31": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.32": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.33": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.34": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.35": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.36": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.37": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.38": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.39": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.4": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.40": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.41": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.42": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.43": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.44": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.45": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.46": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.47": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.48": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.49": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.5": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.50": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.51": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.52": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.53": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.54": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.55": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.56": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.57": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.58": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.59": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.6": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.60": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.61": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.62": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.63": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.64": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.65": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.66": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.67": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.68": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.69": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.7": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.70": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.71": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.72": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.73": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.74": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.75": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.76": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.77": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.78": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.79": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.8": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.80": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.81": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.82": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.83": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.84": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.85": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.86": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.87": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.88": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.89": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.9": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.90": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.91": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.92": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.93": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.94": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.95": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.96": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.97": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.98": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.experts.gate_up_proj_list.99": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.shared_expert.down_proj": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.shared_expert.gate_proj": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 512, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.shared_expert.up_proj": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 512, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.30.mlp.shared_expert_gate": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16, + "out_features": 1, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.0": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.1": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.10": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.100": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.101": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.102": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.103": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.104": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.105": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.106": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.107": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.108": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.109": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.11": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.110": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.111": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.112": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.113": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.114": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.115": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.116": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.117": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.118": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.119": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.12": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.120": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.121": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.122": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.123": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.124": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.125": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.126": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.127": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.128": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.129": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.13": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.130": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.131": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.132": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.133": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.134": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.135": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.136": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.137": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.138": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.139": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.14": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.140": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.141": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.142": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.143": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.144": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.145": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.146": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.147": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.148": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.149": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.15": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.150": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.151": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.152": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.153": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.154": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.155": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.156": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.157": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.158": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.159": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.16": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.160": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.161": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.162": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.163": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.164": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.165": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.166": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.167": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.168": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.169": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.17": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.170": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.171": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.172": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.173": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.174": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.175": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.176": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.177": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.178": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.179": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.18": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.180": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.181": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.182": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.183": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.184": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.185": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.186": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.187": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.188": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.189": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.19": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.190": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.191": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.192": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.193": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.194": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.195": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.196": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.197": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.198": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.199": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.2": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.20": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.200": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.201": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.202": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.203": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.204": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.205": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.206": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.207": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.208": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.209": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.21": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.210": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.211": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.212": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.213": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.214": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.215": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.216": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.217": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.218": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.219": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.22": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.220": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.221": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.222": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.223": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.224": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.225": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.226": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.227": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.228": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.229": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.23": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.230": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.231": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.232": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.233": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.234": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.235": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.236": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.237": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.238": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.239": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.24": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.240": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.241": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.242": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.243": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.244": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.245": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.246": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.247": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.248": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.249": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.25": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.250": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.251": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.252": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.253": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.254": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.255": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.26": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.27": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.28": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.29": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.3": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.30": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.31": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.32": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.33": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.34": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.35": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.36": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.37": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.38": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.39": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.4": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.40": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.41": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.42": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.43": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.44": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.45": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.46": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.47": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.48": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.49": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.5": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.50": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.51": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.52": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.53": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.54": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.55": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.56": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.57": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.58": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.59": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.6": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.60": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.61": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.62": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.63": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.64": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.65": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.66": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.67": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.68": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.69": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.7": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.70": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.71": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.72": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.73": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.74": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.75": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.76": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.77": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.78": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.79": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.8": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.80": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.81": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.82": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.83": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.84": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.85": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.86": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.87": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.88": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.89": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.9": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.90": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.91": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.92": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.93": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.94": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.95": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.96": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.97": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.98": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.down_proj_list.99": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.0": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.1": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.10": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.100": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.101": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.102": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.103": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.104": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.105": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.106": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.107": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.108": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.109": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.11": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.110": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.111": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.112": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.113": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.114": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.115": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.116": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.117": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.118": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.119": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.12": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.120": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.121": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.122": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.123": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.124": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.125": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.126": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.127": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.128": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.129": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.13": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.130": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.131": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.132": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.133": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.134": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.135": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.136": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.137": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.138": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.139": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.14": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.140": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.141": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.142": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.143": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.144": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.145": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.146": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.147": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.148": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.149": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.15": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.150": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.151": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.152": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.153": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.154": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.155": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.156": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.157": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.158": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.159": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.16": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.160": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.161": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.162": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.163": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.164": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.165": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.166": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.167": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.168": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.169": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.17": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.170": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.171": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.172": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.173": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.174": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.175": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.176": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.177": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.178": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.179": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.18": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.180": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.181": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.182": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.183": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.184": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.185": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.186": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.187": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.188": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.189": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.19": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.190": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.191": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.192": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.193": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.194": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.195": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.196": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.197": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.198": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.199": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.2": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.20": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.200": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.201": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.202": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.203": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.204": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.205": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.206": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.207": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.208": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.209": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.21": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.210": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.211": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.212": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.213": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.214": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.215": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.216": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.217": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.218": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.219": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.22": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.220": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.221": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.222": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.223": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.224": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.225": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.226": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.227": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.228": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.229": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.23": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.230": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.231": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.232": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.233": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.234": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.235": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.236": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.237": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.238": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.239": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.24": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.240": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.241": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.242": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.243": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.244": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.245": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.246": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.247": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.248": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.249": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.25": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.250": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.251": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.252": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.253": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.254": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.255": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.26": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.27": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.28": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.29": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.3": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.30": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.31": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.32": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.33": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.34": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.35": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.36": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.37": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.38": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.39": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.4": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.40": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.41": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.42": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.43": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.44": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.45": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.46": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.47": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.48": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.49": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.5": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.50": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.51": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.52": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.53": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.54": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.55": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.56": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.57": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.58": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.59": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.6": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.60": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.61": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.62": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.63": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.64": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.65": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.66": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.67": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.68": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.69": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.7": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.70": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.71": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.72": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.73": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.74": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.75": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.76": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.77": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.78": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.79": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.8": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.80": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.81": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.82": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.83": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.84": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.85": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.86": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.87": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.88": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.89": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.9": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.90": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.91": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.92": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.93": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.94": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.95": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.96": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.97": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.98": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.experts.gate_up_proj_list.99": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.shared_expert.down_proj": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.shared_expert.gate_proj": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 512, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.shared_expert.up_proj": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 512, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.mlp.shared_expert_gate": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16, + "out_features": 1, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.self_attn.k_proj": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 512, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.self_attn.o_proj": { + "group_size": 128, + "in_features": 4096, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 65536, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 4096, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.self_attn.q_proj": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 131072, + "out_features": 8192, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.31.self_attn.v_proj": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 512, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.linear_attn.in_proj_a": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 512, + "out_features": 32, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.linear_attn.in_proj_b": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 512, + "out_features": 32, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.linear_attn.in_proj_qkv": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 131072, + "out_features": 8192, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.linear_attn.in_proj_z": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 65536, + "out_features": 4096, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.linear_attn.out_proj": { + "group_size": 128, + "in_features": 4096, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 65536, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 4096, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.0": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.1": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.10": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.100": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.101": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.102": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.103": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.104": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.105": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.106": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.107": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.108": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.109": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.11": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.110": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.111": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.112": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.113": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.114": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.115": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.116": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.117": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.118": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.119": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.12": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.120": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.121": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.122": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.123": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.124": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.125": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.126": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.127": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.128": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.129": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.13": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.130": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.131": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.132": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.133": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.134": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.135": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.136": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.137": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.138": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.139": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.14": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.140": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.141": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.142": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.143": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.144": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.145": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.146": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.147": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.148": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.149": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.15": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.150": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.151": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.152": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.153": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.154": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.155": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.156": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.157": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.158": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.159": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.16": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.160": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.161": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.162": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.163": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.164": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.165": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.166": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.167": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.168": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.169": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.17": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.170": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.171": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.172": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.173": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.174": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.175": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.176": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.177": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.178": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.179": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.18": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.180": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.181": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.182": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.183": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.184": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.185": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.186": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.187": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.188": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.189": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.19": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.190": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.191": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.192": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.193": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.194": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.195": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.196": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.197": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.198": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.199": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.2": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.20": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.200": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.201": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.202": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.203": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.204": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.205": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.206": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.207": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.208": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.209": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.21": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.210": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.211": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.212": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.213": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.214": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.215": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.216": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.217": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.218": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.219": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.22": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.220": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.221": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.222": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.223": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.224": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.225": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.226": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.227": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.228": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.229": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.23": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.230": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.231": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.232": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.233": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.234": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.235": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.236": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.237": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.238": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.239": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.24": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.240": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.241": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.242": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.243": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.244": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.245": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.246": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.247": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.248": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.249": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.25": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.250": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.251": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.252": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.253": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.254": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.255": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.26": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.27": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.28": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.29": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.3": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.30": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.31": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.32": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.33": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.34": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.35": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.36": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.37": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.38": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.39": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.4": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.40": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.41": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.42": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.43": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.44": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.45": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.46": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.47": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.48": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.49": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.5": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.50": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.51": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.52": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.53": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.54": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.55": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.56": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.57": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.58": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.59": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.6": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.60": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.61": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.62": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.63": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.64": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.65": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.66": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.67": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.68": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.69": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.7": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.70": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.71": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.72": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.73": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.74": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.75": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.76": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.77": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.78": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.79": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.8": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.80": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.81": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.82": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.83": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.84": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.85": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.86": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.87": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.88": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.89": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.9": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.90": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.91": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.92": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.93": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.94": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.95": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.96": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.97": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.98": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.down_proj_list.99": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.0": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.1": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.10": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.100": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.101": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.102": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.103": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.104": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.105": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.106": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.107": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.108": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.109": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.11": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.110": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.111": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.112": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.113": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.114": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.115": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.116": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.117": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.118": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.119": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.12": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.120": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.121": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.122": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.123": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.124": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.125": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.126": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.127": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.128": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.129": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.13": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.130": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.131": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.132": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.133": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.134": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.135": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.136": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.137": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.138": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.139": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.14": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.140": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.141": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.142": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.143": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.144": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.145": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.146": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.147": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.148": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.149": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.15": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.150": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.151": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.152": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.153": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.154": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.155": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.156": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.157": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.158": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.159": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.16": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.160": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.161": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.162": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.163": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.164": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.165": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.166": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.167": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.168": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.169": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.17": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.170": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.171": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.172": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.173": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.174": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.175": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.176": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.177": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.178": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.179": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.18": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.180": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.181": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.182": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.183": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.184": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.185": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.186": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.187": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.188": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.189": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.19": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.190": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.191": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.192": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.193": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.194": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.195": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.196": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.197": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.198": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.199": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.2": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.20": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.200": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.201": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.202": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.203": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.204": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.205": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.206": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.207": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.208": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.209": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.21": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.210": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.211": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.212": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.213": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.214": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.215": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.216": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.217": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.218": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.219": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.22": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.220": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.221": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.222": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.223": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.224": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.225": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.226": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.227": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.228": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.229": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.23": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.230": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.231": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.232": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.233": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.234": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.235": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.236": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.237": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.238": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.239": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.24": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.240": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.241": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.242": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.243": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.244": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.245": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.246": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.247": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.248": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.249": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.25": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.250": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.251": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.252": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.253": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.254": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.255": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.26": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.27": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.28": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.29": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.3": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.30": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.31": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.32": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.33": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.34": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.35": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.36": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.37": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.38": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.39": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.4": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.40": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.41": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.42": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.43": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.44": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.45": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.46": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.47": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.48": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.49": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.5": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.50": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.51": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.52": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.53": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.54": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.55": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.56": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.57": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.58": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.59": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.6": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.60": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.61": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.62": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.63": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.64": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.65": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.66": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.67": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.68": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.69": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.7": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.70": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.71": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.72": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.73": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.74": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.75": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.76": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.77": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.78": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.79": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.8": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.80": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.81": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.82": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.83": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.84": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.85": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.86": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.87": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.88": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.89": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.9": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.90": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.91": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.92": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.93": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.94": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.95": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.96": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.97": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.98": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.experts.gate_up_proj_list.99": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.shared_expert.down_proj": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.shared_expert.gate_proj": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 512, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.shared_expert.up_proj": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 512, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.32.mlp.shared_expert_gate": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16, + "out_features": 1, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.linear_attn.in_proj_a": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 512, + "out_features": 32, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.linear_attn.in_proj_b": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 512, + "out_features": 32, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.linear_attn.in_proj_qkv": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 131072, + "out_features": 8192, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.linear_attn.in_proj_z": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 65536, + "out_features": 4096, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.linear_attn.out_proj": { + "group_size": 128, + "in_features": 4096, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 65536, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 4096, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.0": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.1": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.10": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.100": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.101": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.102": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.103": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.104": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.105": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.106": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.107": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.108": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.109": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.11": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.110": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.111": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.112": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.113": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.114": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.115": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.116": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.117": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.118": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.119": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.12": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.120": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.121": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.122": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.123": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.124": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.125": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.126": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.127": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.128": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.129": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.13": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.130": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.131": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.132": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.133": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.134": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.135": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.136": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.137": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.138": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.139": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.14": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.140": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.141": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.142": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.143": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.144": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.145": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.146": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.147": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.148": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.149": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.15": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.150": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.151": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.152": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.153": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.154": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.155": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.156": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.157": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.158": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.159": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.16": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.160": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.161": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.162": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.163": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.164": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.165": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.166": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.167": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.168": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.169": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.17": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.170": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.171": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.172": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.173": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.174": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.175": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.176": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.177": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.178": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.179": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.18": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.180": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.181": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.182": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.183": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.184": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.185": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.186": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.187": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.188": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.189": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.19": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.190": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.191": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.192": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.193": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.194": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.195": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.196": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.197": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.198": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.199": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.2": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.20": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.200": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.201": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.202": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.203": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.204": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.205": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.206": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.207": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.208": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.209": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.21": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.210": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.211": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.212": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.213": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.214": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.215": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.216": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.217": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.218": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.219": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.22": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.220": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.221": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.222": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.223": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.224": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.225": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.226": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.227": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.228": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.229": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.23": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.230": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.231": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.232": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.233": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.234": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.235": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.236": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.237": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.238": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.239": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.24": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.240": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.241": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.242": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.243": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.244": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.245": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.246": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.247": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.248": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.249": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.25": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.250": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.251": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.252": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.253": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.254": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.255": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.26": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.27": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.28": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.29": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.3": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.30": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.31": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.32": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.33": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.34": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.35": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.36": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.37": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.38": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.39": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.4": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.40": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.41": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.42": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.43": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.44": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.45": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.46": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.47": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.48": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.49": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.5": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.50": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.51": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.52": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.53": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.54": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.55": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.56": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.57": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.58": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.59": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.6": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.60": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.61": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.62": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.63": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.64": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.65": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.66": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.67": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.68": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.69": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.7": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.70": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.71": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.72": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.73": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.74": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.75": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.76": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.77": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.78": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.79": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.8": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.80": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.81": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.82": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.83": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.84": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.85": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.86": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.87": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.88": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.89": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.9": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.90": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.91": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.92": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.93": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.94": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.95": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.96": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.97": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.98": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.down_proj_list.99": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.0": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.1": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.10": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.100": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.101": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.102": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.103": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.104": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.105": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.106": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.107": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.108": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.109": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.11": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.110": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.111": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.112": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.113": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.114": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.115": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.116": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.117": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.118": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.119": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.12": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.120": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.121": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.122": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.123": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.124": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.125": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.126": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.127": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.128": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.129": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.13": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.130": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.131": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.132": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.133": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.134": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.135": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.136": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.137": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.138": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.139": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.14": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.140": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.141": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.142": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.143": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.144": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.145": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.146": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.147": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.148": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.149": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.15": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.150": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.151": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.152": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.153": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.154": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.155": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.156": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.157": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.158": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.159": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.16": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.160": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.161": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.162": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.163": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.164": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.165": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.166": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.167": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.168": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.169": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.17": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.170": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.171": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.172": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.173": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.174": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.175": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.176": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.177": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.178": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.179": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.18": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.180": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.181": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.182": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.183": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.184": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.185": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.186": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.187": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.188": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.189": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.19": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.190": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.191": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.192": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.193": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.194": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.195": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.196": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.197": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.198": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.199": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.2": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.20": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.200": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.201": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.202": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.203": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.204": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.205": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.206": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.207": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.208": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.209": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.21": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.210": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.211": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.212": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.213": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.214": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.215": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.216": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.217": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.218": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.219": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.22": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.220": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.221": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.222": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.223": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.224": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.225": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.226": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.227": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.228": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.229": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.23": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.230": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.231": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.232": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.233": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.234": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.235": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.236": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.237": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.238": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.239": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.24": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.240": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.241": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.242": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.243": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.244": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.245": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.246": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.247": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.248": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.249": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.25": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.250": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.251": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.252": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.253": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.254": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.255": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.26": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.27": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.28": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.29": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.3": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.30": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.31": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.32": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.33": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.34": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.35": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.36": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.37": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.38": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.39": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.4": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.40": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.41": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.42": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.43": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.44": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.45": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.46": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.47": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.48": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.49": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.5": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.50": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.51": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.52": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.53": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.54": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.55": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.56": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.57": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.58": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.59": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.6": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.60": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.61": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.62": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.63": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.64": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.65": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.66": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.67": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.68": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.69": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.7": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.70": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.71": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.72": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.73": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.74": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.75": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.76": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.77": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.78": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.79": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.8": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.80": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.81": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.82": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.83": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.84": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.85": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.86": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.87": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.88": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.89": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.9": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.90": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.91": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.92": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.93": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.94": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.95": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.96": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.97": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.98": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.experts.gate_up_proj_list.99": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.shared_expert.down_proj": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.shared_expert.gate_proj": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 512, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.shared_expert.up_proj": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 512, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.33.mlp.shared_expert_gate": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16, + "out_features": 1, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.linear_attn.in_proj_a": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 512, + "out_features": 32, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.linear_attn.in_proj_b": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 512, + "out_features": 32, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.linear_attn.in_proj_qkv": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 131072, + "out_features": 8192, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.linear_attn.in_proj_z": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 65536, + "out_features": 4096, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.linear_attn.out_proj": { + "group_size": 128, + "in_features": 4096, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 65536, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 4096, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.0": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.1": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.10": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.100": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.101": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.102": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.103": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.104": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.105": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.106": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.107": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.108": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.109": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.11": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.110": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.111": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.112": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.113": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.114": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.115": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.116": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.117": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.118": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.119": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.12": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.120": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.121": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.122": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.123": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.124": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.125": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.126": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.127": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.128": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.129": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.13": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.130": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.131": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.132": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.133": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.134": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.135": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.136": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.137": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.138": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.139": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.14": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.140": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.141": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.142": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.143": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.144": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.145": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.146": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.147": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.148": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.149": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.15": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.150": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.151": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.152": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.153": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.154": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.155": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.156": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.157": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.158": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.159": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.16": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.160": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.161": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.162": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.163": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.164": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.165": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.166": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.167": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.168": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.169": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.17": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.170": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.171": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.172": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.173": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.174": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.175": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.176": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.177": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.178": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.179": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.18": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.180": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.181": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.182": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.183": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.184": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.185": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.186": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.187": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.188": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.189": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.19": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.190": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.191": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.192": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.193": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.194": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.195": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.196": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.197": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.198": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.199": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.2": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.20": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.200": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.201": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.202": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.203": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.204": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.205": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.206": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.207": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.208": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.209": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.21": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.210": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.211": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.212": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.213": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.214": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.215": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.216": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.217": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.218": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.219": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.22": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.220": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.221": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.222": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.223": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.224": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.225": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.226": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.227": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.228": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.229": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.23": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.230": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.231": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.232": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.233": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.234": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.235": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.236": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.237": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.238": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.239": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.24": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.240": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.241": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.242": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.243": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.244": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.245": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.246": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.247": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.248": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.249": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.25": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.250": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.251": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.252": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.253": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.254": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.255": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.26": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.27": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.28": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.29": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.3": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.30": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.31": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.32": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.33": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.34": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.35": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.36": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.37": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.38": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.39": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.4": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.40": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.41": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.42": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.43": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.44": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.45": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.46": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.47": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.48": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.49": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.5": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.50": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.51": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.52": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.53": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.54": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.55": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.56": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.57": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.58": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.59": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.6": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.60": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.61": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.62": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.63": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.64": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.65": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.66": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.67": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.68": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.69": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.7": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.70": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.71": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.72": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.73": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.74": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.75": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.76": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.77": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.78": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.79": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.8": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.80": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.81": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.82": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.83": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.84": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.85": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.86": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.87": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.88": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.89": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.9": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.90": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.91": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.92": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.93": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.94": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.95": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.96": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.97": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.98": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.down_proj_list.99": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.0": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.1": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.10": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.100": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.101": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.102": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.103": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.104": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.105": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.106": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.107": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.108": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.109": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.11": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.110": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.111": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.112": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.113": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.114": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.115": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.116": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.117": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.118": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.119": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.12": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.120": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.121": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.122": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.123": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.124": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.125": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.126": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.127": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.128": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.129": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.13": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.130": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.131": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.132": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.133": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.134": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.135": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.136": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.137": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.138": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.139": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.14": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.140": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.141": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.142": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.143": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.144": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.145": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.146": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.147": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.148": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.149": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.15": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.150": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.151": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.152": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.153": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.154": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.155": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.156": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.157": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.158": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.159": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.16": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.160": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.161": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.162": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.163": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.164": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.165": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.166": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.167": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.168": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.169": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.17": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.170": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.171": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.172": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.173": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.174": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.175": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.176": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.177": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.178": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.179": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.18": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.180": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.181": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.182": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.183": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.184": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.185": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.186": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.187": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.188": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.189": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.19": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.190": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.191": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.192": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.193": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.194": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.195": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.196": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.197": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.198": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.199": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.2": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.20": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.200": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.201": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.202": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.203": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.204": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.205": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.206": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.207": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.208": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.209": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.21": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.210": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.211": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.212": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.213": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.214": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.215": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.216": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.217": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.218": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.219": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.22": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.220": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.221": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.222": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.223": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.224": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.225": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.226": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.227": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.228": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.229": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.23": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.230": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.231": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.232": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.233": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.234": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.235": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.236": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.237": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.238": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.239": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.24": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.240": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.241": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.242": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.243": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.244": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.245": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.246": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.247": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.248": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.249": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.25": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.250": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.251": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.252": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.253": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.254": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.255": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.26": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.27": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.28": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.29": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.3": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.30": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.31": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.32": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.33": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.34": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.35": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.36": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.37": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.38": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.39": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.4": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.40": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.41": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.42": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.43": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.44": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.45": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.46": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.47": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.48": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.49": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.5": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.50": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.51": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.52": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.53": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.54": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.55": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.56": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.57": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.58": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.59": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.6": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.60": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.61": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.62": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.63": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.64": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.65": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.66": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.67": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.68": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.69": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.7": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.70": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.71": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.72": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.73": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.74": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.75": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.76": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.77": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.78": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.79": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.8": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.80": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.81": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.82": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.83": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.84": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.85": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.86": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.87": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.88": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.89": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.9": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.90": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.91": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.92": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.93": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.94": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.95": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.96": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.97": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.98": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.experts.gate_up_proj_list.99": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.shared_expert.down_proj": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.shared_expert.gate_proj": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 512, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.shared_expert.up_proj": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 512, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.34.mlp.shared_expert_gate": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16, + "out_features": 1, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.0": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.1": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.10": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.100": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.101": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.102": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.103": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.104": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.105": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.106": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.107": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.108": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.109": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.11": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.110": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.111": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.112": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.113": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.114": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.115": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.116": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.117": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.118": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.119": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.12": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.120": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.121": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.122": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.123": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.124": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.125": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.126": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.127": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.128": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.129": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.13": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.130": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.131": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.132": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.133": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.134": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.135": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.136": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.137": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.138": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.139": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.14": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.140": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.141": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.142": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.143": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.144": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.145": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.146": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.147": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.148": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.149": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.15": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.150": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.151": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.152": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.153": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.154": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.155": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.156": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.157": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.158": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.159": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.16": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.160": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.161": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.162": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.163": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.164": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.165": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.166": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.167": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.168": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.169": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.17": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.170": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.171": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.172": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.173": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.174": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.175": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.176": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.177": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.178": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.179": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.18": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.180": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.181": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.182": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.183": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.184": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.185": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.186": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.187": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.188": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.189": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.19": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.190": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.191": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.192": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.193": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.194": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.195": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.196": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.197": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.198": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.199": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.2": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.20": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.200": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.201": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.202": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.203": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.204": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.205": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.206": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.207": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.208": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.209": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.21": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.210": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.211": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.212": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.213": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.214": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.215": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.216": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.217": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.218": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.219": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.22": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.220": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.221": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.222": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.223": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.224": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.225": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.226": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.227": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.228": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.229": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.23": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.230": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.231": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.232": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.233": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.234": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.235": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.236": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.237": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.238": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.239": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.24": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.240": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.241": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.242": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.243": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.244": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.245": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.246": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.247": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.248": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.249": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.25": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.250": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.251": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.252": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.253": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.254": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.255": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.26": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.27": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.28": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.29": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.3": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.30": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.31": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.32": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.33": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.34": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.35": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.36": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.37": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.38": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.39": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.4": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.40": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.41": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.42": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.43": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.44": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.45": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.46": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.47": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.48": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.49": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.5": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.50": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.51": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.52": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.53": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.54": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.55": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.56": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.57": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.58": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.59": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.6": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.60": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.61": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.62": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.63": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.64": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.65": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.66": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.67": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.68": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.69": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.7": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.70": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.71": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.72": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.73": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.74": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.75": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.76": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.77": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.78": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.79": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.8": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.80": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.81": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.82": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.83": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.84": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.85": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.86": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.87": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.88": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.89": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.9": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.90": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.91": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.92": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.93": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.94": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.95": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.96": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.97": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.98": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.down_proj_list.99": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.0": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.1": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.10": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.100": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.101": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.102": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.103": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.104": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.105": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.106": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.107": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.108": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.109": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.11": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.110": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.111": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.112": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.113": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.114": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.115": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.116": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.117": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.118": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.119": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.12": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.120": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.121": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.122": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.123": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.124": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.125": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.126": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.127": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.128": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.129": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.13": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.130": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.131": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.132": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.133": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.134": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.135": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.136": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.137": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.138": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.139": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.14": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.140": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.141": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.142": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.143": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.144": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.145": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.146": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.147": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.148": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.149": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.15": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.150": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.151": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.152": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.153": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.154": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.155": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.156": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.157": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.158": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.159": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.16": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.160": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.161": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.162": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.163": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.164": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.165": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.166": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.167": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.168": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.169": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.17": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.170": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.171": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.172": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.173": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.174": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.175": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.176": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.177": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.178": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.179": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.18": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.180": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.181": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.182": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.183": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.184": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.185": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.186": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.187": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.188": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.189": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.19": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.190": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.191": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.192": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.193": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.194": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.195": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.196": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.197": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.198": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.199": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.2": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.20": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.200": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.201": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.202": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.203": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.204": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.205": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.206": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.207": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.208": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.209": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.21": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.210": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.211": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.212": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.213": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.214": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.215": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.216": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.217": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.218": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.219": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.22": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.220": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.221": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.222": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.223": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.224": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.225": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.226": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.227": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.228": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.229": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.23": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.230": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.231": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.232": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.233": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.234": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.235": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.236": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.237": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.238": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.239": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.24": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.240": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.241": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.242": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.243": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.244": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.245": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.246": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.247": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.248": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.249": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.25": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.250": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.251": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.252": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.253": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.254": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.255": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.26": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.27": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.28": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.29": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.3": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.30": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.31": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.32": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.33": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.34": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.35": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.36": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.37": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.38": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.39": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.4": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.40": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.41": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.42": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.43": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.44": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.45": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.46": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.47": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.48": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.49": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.5": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.50": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.51": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.52": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.53": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.54": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.55": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.56": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.57": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.58": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.59": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.6": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.60": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.61": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.62": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.63": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.64": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.65": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.66": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.67": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.68": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.69": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.7": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.70": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.71": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.72": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.73": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.74": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.75": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.76": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.77": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.78": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.79": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.8": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.80": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.81": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.82": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.83": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.84": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.85": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.86": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.87": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.88": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.89": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.9": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.90": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.91": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.92": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.93": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.94": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.95": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.96": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.97": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.98": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.experts.gate_up_proj_list.99": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.shared_expert.down_proj": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.shared_expert.gate_proj": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 512, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.shared_expert.up_proj": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 512, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.mlp.shared_expert_gate": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16, + "out_features": 1, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.self_attn.k_proj": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 512, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.self_attn.o_proj": { + "group_size": 128, + "in_features": 4096, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 65536, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 4096, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.self_attn.q_proj": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 131072, + "out_features": 8192, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.35.self_attn.v_proj": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 512, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.linear_attn.in_proj_a": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 512, + "out_features": 32, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.linear_attn.in_proj_b": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 512, + "out_features": 32, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.linear_attn.in_proj_qkv": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 131072, + "out_features": 8192, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.linear_attn.in_proj_z": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 65536, + "out_features": 4096, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.linear_attn.out_proj": { + "group_size": 128, + "in_features": 4096, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 65536, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 4096, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.0": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.1": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.10": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.100": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.101": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.102": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.103": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.104": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.105": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.106": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.107": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.108": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.109": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.11": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.110": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.111": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.112": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.113": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.114": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.115": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.116": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.117": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.118": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.119": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.12": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.120": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.121": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.122": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.123": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.124": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.125": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.126": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.127": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.128": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.129": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.13": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.130": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.131": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.132": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.133": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.134": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.135": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.136": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.137": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.138": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.139": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.14": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.140": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.141": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.142": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.143": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.144": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.145": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.146": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.147": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.148": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.149": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.15": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.150": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.151": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.152": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.153": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.154": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.155": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.156": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.157": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.158": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.159": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.16": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.160": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.161": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.162": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.163": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.164": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.165": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.166": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.167": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.168": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.169": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.17": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.170": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.171": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.172": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.173": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.174": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.175": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.176": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.177": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.178": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.179": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.18": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.180": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.181": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.182": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.183": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.184": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.185": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.186": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.187": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.188": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.189": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.19": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.190": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.191": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.192": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.193": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.194": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.195": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.196": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.197": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.198": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.199": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.2": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.20": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.200": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.201": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.202": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.203": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.204": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.205": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.206": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.207": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.208": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.209": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.21": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.210": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.211": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.212": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.213": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.214": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.215": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.216": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.217": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.218": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.219": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.22": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.220": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.221": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.222": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.223": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.224": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.225": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.226": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.227": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.228": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.229": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.23": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.230": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.231": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.232": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.233": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.234": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.235": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.236": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.237": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.238": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.239": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.24": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.240": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.241": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.242": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.243": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.244": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.245": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.246": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.247": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.248": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.249": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.25": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.250": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.251": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.252": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.253": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.254": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.255": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.26": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.27": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.28": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.29": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.3": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.30": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.31": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.32": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.33": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.34": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.35": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.36": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.37": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.38": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.39": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.4": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.40": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.41": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.42": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.43": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.44": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.45": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.46": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.47": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.48": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.49": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.5": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.50": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.51": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.52": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.53": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.54": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.55": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.56": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.57": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.58": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.59": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.6": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.60": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.61": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.62": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.63": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.64": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.65": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.66": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.67": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.68": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.69": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.7": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.70": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.71": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.72": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.73": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.74": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.75": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.76": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.77": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.78": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.79": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.8": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.80": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.81": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.82": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.83": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.84": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.85": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.86": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.87": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.88": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.89": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.9": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.90": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.91": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.92": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.93": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.94": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.95": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.96": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.97": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.98": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.down_proj_list.99": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.0": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.1": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.10": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.100": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.101": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.102": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.103": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.104": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.105": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.106": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.107": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.108": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.109": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.11": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.110": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.111": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.112": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.113": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.114": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.115": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.116": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.117": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.118": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.119": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.12": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.120": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.121": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.122": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.123": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.124": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.125": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.126": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.127": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.128": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.129": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.13": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.130": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.131": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.132": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.133": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.134": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.135": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.136": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.137": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.138": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.139": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.14": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.140": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.141": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.142": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.143": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.144": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.145": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.146": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.147": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.148": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.149": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.15": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.150": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.151": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.152": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.153": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.154": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.155": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.156": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.157": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.158": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.159": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.16": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.160": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.161": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.162": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.163": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.164": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.165": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.166": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.167": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.168": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.169": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.17": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.170": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.171": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.172": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.173": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.174": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.175": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.176": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.177": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.178": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.179": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.18": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.180": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.181": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.182": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.183": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.184": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.185": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.186": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.187": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.188": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.189": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.19": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.190": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.191": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.192": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.193": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.194": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.195": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.196": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.197": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.198": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.199": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.2": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.20": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.200": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.201": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.202": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.203": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.204": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.205": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.206": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.207": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.208": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.209": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.21": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.210": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.211": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.212": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.213": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.214": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.215": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.216": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.217": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.218": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.219": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.22": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.220": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.221": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.222": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.223": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.224": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.225": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.226": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.227": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.228": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.229": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.23": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.230": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.231": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.232": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.233": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.234": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.235": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.236": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.237": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.238": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.239": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.24": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.240": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.241": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.242": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.243": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.244": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.245": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.246": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.247": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.248": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.249": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.25": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.250": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.251": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.252": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.253": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.254": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.255": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.26": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.27": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.28": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.29": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.3": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.30": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.31": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.32": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.33": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.34": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.35": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.36": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.37": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.38": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.39": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.4": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.40": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.41": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.42": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.43": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.44": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.45": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.46": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.47": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.48": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.49": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.5": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.50": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.51": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.52": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.53": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.54": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.55": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.56": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.57": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.58": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.59": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.6": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.60": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.61": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.62": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.63": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.64": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.65": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.66": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.67": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.68": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.69": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.7": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.70": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.71": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.72": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.73": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.74": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.75": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.76": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.77": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.78": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.79": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.8": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.80": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.81": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.82": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.83": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.84": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.85": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.86": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.87": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.88": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.89": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.9": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.90": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.91": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.92": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.93": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.94": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.95": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.96": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.97": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.98": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.experts.gate_up_proj_list.99": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.shared_expert.down_proj": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.shared_expert.gate_proj": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 512, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.shared_expert.up_proj": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 512, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.36.mlp.shared_expert_gate": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16, + "out_features": 1, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.linear_attn.in_proj_a": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 512, + "out_features": 32, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.linear_attn.in_proj_b": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 512, + "out_features": 32, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.linear_attn.in_proj_qkv": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 131072, + "out_features": 8192, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.linear_attn.in_proj_z": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 65536, + "out_features": 4096, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.linear_attn.out_proj": { + "group_size": 128, + "in_features": 4096, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 65536, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 4096, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.0": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.1": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.10": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.100": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.101": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.102": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.103": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.104": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.105": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.106": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.107": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.108": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.109": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.11": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.110": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.111": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.112": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.113": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.114": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.115": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.116": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.117": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.118": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.119": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.12": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.120": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.121": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.122": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.123": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.124": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.125": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.126": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.127": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.128": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.129": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.13": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.130": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.131": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.132": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.133": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.134": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.135": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.136": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.137": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.138": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.139": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.14": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.140": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.141": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.142": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.143": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.144": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.145": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.146": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.147": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.148": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.149": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.15": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.150": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.151": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.152": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.153": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.154": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.155": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.156": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.157": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.158": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.159": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.16": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.160": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.161": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.162": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.163": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.164": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.165": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.166": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.167": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.168": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.169": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.17": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.170": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.171": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.172": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.173": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.174": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.175": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.176": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.177": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.178": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.179": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.18": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.180": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.181": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.182": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.183": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.184": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.185": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.186": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.187": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.188": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.189": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.19": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.190": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.191": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.192": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.193": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.194": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.195": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.196": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.197": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.198": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.199": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.2": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.20": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.200": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.201": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.202": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.203": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.204": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.205": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.206": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.207": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.208": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.209": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.21": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.210": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.211": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.212": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.213": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.214": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.215": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.216": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.217": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.218": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.219": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.22": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.220": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.221": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.222": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.223": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.224": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.225": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.226": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.227": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.228": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.229": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.23": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.230": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.231": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.232": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.233": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.234": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.235": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.236": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.237": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.238": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.239": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.24": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.240": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.241": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.242": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.243": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.244": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.245": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.246": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.247": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.248": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.249": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.25": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.250": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.251": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.252": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.253": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.254": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.255": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.26": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.27": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.28": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.29": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.3": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.30": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.31": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.32": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.33": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.34": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.35": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.36": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.37": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.38": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.39": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.4": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.40": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.41": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.42": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.43": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.44": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.45": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.46": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.47": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.48": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.49": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.5": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.50": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.51": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.52": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.53": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.54": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.55": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.56": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.57": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.58": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.59": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.6": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.60": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.61": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.62": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.63": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.64": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.65": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.66": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.67": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.68": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.69": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.7": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.70": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.71": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.72": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.73": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.74": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.75": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.76": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.77": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.78": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.79": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.8": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.80": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.81": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.82": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.83": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.84": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.85": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.86": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.87": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.88": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.89": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.9": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.90": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.91": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.92": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.93": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.94": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.95": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.96": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.97": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.98": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.down_proj_list.99": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.0": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.1": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.10": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.100": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.101": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.102": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.103": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.104": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.105": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.106": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.107": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.108": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.109": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.11": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.110": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.111": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.112": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.113": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.114": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.115": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.116": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.117": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.118": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.119": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.12": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.120": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.121": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.122": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.123": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.124": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.125": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.126": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.127": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.128": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.129": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.13": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.130": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.131": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.132": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.133": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.134": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.135": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.136": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.137": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.138": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.139": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.14": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.140": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.141": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.142": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.143": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.144": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.145": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.146": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.147": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.148": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.149": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.15": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.150": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.151": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.152": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.153": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.154": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.155": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.156": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.157": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.158": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.159": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.16": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.160": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.161": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.162": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.163": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.164": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.165": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.166": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.167": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.168": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.169": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.17": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.170": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.171": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.172": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.173": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.174": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.175": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.176": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.177": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.178": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.179": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.18": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.180": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.181": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.182": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.183": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.184": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.185": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.186": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.187": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.188": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.189": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.19": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.190": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.191": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.192": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.193": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.194": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.195": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.196": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.197": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.198": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.199": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.2": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.20": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.200": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.201": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.202": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.203": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.204": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.205": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.206": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.207": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.208": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.209": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.21": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.210": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.211": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.212": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.213": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.214": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.215": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.216": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.217": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.218": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.219": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.22": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.220": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.221": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.222": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.223": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.224": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.225": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.226": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.227": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.228": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.229": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.23": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.230": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.231": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.232": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.233": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.234": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.235": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.236": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.237": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.238": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.239": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.24": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.240": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.241": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.242": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.243": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.244": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.245": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.246": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.247": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.248": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.249": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.25": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.250": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.251": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.252": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.253": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.254": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.255": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.26": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.27": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.28": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.29": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.3": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.30": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.31": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.32": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.33": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.34": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.35": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.36": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.37": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.38": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.39": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.4": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.40": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.41": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.42": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.43": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.44": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.45": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.46": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.47": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.48": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.49": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.5": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.50": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.51": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.52": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.53": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.54": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.55": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.56": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.57": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.58": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.59": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.6": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.60": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.61": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.62": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.63": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.64": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.65": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.66": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.67": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.68": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.69": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.7": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.70": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.71": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.72": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.73": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.74": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.75": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.76": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.77": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.78": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.79": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.8": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.80": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.81": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.82": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.83": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.84": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.85": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.86": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.87": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.88": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.89": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.9": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.90": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.91": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.92": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.93": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.94": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.95": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.96": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.97": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.98": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.experts.gate_up_proj_list.99": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.shared_expert.down_proj": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.shared_expert.gate_proj": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 512, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.shared_expert.up_proj": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 512, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.37.mlp.shared_expert_gate": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16, + "out_features": 1, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.linear_attn.in_proj_a": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 512, + "out_features": 32, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.linear_attn.in_proj_b": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 512, + "out_features": 32, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.linear_attn.in_proj_qkv": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 131072, + "out_features": 8192, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.linear_attn.in_proj_z": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 65536, + "out_features": 4096, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.linear_attn.out_proj": { + "group_size": 128, + "in_features": 4096, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 65536, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 4096, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.0": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.1": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.10": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.100": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.101": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.102": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.103": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.104": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.105": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.106": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.107": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.108": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.109": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.11": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.110": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.111": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.112": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.113": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.114": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.115": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.116": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.117": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.118": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.119": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.12": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.120": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.121": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.122": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.123": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.124": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.125": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.126": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.127": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.128": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.129": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.13": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.130": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.131": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.132": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.133": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.134": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.135": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.136": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.137": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.138": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.139": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.14": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.140": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.141": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.142": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.143": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.144": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.145": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.146": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.147": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.148": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.149": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.15": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.150": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.151": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.152": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.153": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.154": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.155": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.156": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.157": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.158": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.159": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.16": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.160": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.161": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.162": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.163": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.164": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.165": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.166": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.167": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.168": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.169": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.17": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.170": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.171": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.172": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.173": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.174": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.175": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.176": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.177": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.178": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.179": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.18": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.180": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.181": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.182": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.183": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.184": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.185": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.186": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.187": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.188": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.189": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.19": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.190": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.191": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.192": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.193": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.194": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.195": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.196": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.197": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.198": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.199": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.2": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.20": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.200": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.201": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.202": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.203": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.204": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.205": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.206": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.207": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.208": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.209": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.21": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.210": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.211": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.212": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.213": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.214": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.215": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.216": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.217": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.218": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.219": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.22": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.220": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.221": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.222": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.223": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.224": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.225": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.226": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.227": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.228": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.229": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.23": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.230": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.231": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.232": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.233": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.234": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.235": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.236": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.237": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.238": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.239": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.24": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.240": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.241": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.242": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.243": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.244": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.245": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.246": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.247": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.248": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.249": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.25": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.250": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.251": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.252": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.253": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.254": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.255": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.26": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.27": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.28": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.29": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.3": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.30": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.31": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.32": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.33": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.34": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.35": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.36": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.37": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.38": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.39": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.4": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.40": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.41": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.42": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.43": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.44": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.45": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.46": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.47": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.48": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.49": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.5": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.50": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.51": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.52": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.53": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.54": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.55": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.56": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.57": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.58": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.59": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.6": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.60": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.61": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.62": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.63": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.64": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.65": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.66": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.67": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.68": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.69": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.7": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.70": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.71": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.72": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.73": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.74": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.75": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.76": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.77": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.78": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.79": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.8": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.80": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.81": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.82": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.83": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.84": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.85": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.86": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.87": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.88": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.89": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.9": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.90": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.91": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.92": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.93": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.94": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.95": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.96": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.97": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.98": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.down_proj_list.99": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.0": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.1": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.10": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.100": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.101": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.102": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.103": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.104": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.105": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.106": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.107": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.108": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.109": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.11": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.110": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.111": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.112": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.113": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.114": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.115": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.116": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.117": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.118": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.119": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.12": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.120": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.121": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.122": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.123": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.124": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.125": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.126": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.127": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.128": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.129": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.13": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.130": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.131": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.132": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.133": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.134": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.135": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.136": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.137": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.138": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.139": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.14": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.140": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.141": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.142": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.143": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.144": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.145": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.146": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.147": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.148": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.149": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.15": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.150": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.151": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.152": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.153": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.154": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.155": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.156": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.157": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.158": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.159": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.16": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.160": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.161": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.162": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.163": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.164": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.165": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.166": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.167": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.168": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.169": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.17": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.170": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.171": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.172": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.173": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.174": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.175": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.176": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.177": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.178": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.179": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.18": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.180": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.181": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.182": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.183": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.184": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.185": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.186": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.187": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.188": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.189": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.19": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.190": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.191": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.192": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.193": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.194": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.195": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.196": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.197": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.198": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.199": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.2": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.20": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.200": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.201": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.202": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.203": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.204": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.205": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.206": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.207": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.208": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.209": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.21": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.210": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.211": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.212": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.213": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.214": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.215": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.216": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.217": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.218": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.219": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.22": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.220": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.221": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.222": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.223": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.224": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.225": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.226": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.227": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.228": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.229": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.23": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.230": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.231": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.232": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.233": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.234": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.235": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.236": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.237": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.238": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.239": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.24": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.240": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.241": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.242": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.243": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.244": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.245": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.246": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.247": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.248": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.249": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.25": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.250": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.251": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.252": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.253": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.254": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.255": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.26": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.27": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.28": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.29": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.3": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.30": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.31": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.32": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.33": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.34": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.35": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.36": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.37": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.38": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.39": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.4": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.40": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.41": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.42": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.43": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.44": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.45": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.46": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.47": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.48": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.49": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.5": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.50": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.51": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.52": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.53": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.54": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.55": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.56": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.57": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.58": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.59": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.6": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.60": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.61": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.62": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.63": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.64": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.65": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.66": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.67": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.68": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.69": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.7": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.70": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.71": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.72": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.73": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.74": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.75": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.76": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.77": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.78": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.79": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.8": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.80": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.81": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.82": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.83": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.84": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.85": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.86": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.87": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.88": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.89": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.9": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.90": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.91": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.92": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.93": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.94": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.95": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.96": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.97": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.98": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.experts.gate_up_proj_list.99": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.shared_expert.down_proj": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.shared_expert.gate_proj": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 512, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.shared_expert.up_proj": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 512, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.38.mlp.shared_expert_gate": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16, + "out_features": 1, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.0": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.1": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.10": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.100": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.101": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.102": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.103": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.104": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.105": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.106": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.107": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.108": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.109": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.11": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.110": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.111": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.112": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.113": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.114": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.115": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.116": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.117": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.118": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.119": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.12": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.120": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.121": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.122": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.123": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.124": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.125": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.126": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.127": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.128": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.129": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.13": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.130": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.131": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.132": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.133": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.134": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.135": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.136": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.137": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.138": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.139": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.14": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.140": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.141": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.142": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.143": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.144": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.145": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.146": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.147": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.148": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.149": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.15": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.150": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.151": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.152": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.153": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.154": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.155": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.156": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.157": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.158": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.159": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.16": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.160": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.161": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.162": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.163": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.164": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.165": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.166": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.167": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.168": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.169": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.17": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.170": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.171": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.172": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.173": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.174": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.175": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.176": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.177": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.178": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.179": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.18": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.180": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.181": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.182": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.183": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.184": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.185": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.186": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.187": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.188": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.189": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.19": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.190": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.191": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.192": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.193": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.194": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.195": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.196": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.197": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.198": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.199": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.2": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.20": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.200": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.201": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.202": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.203": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.204": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.205": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.206": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.207": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.208": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.209": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.21": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.210": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.211": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.212": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.213": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.214": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.215": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.216": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.217": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.218": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.219": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.22": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.220": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.221": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.222": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.223": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.224": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.225": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.226": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.227": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.228": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.229": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.23": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.230": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.231": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.232": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.233": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.234": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.235": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.236": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.237": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.238": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.239": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.24": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.240": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.241": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.242": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.243": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.244": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.245": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.246": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.247": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.248": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.249": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.25": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.250": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.251": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.252": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.253": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.254": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.255": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.26": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.27": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.28": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.29": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.3": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.30": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.31": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.32": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.33": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.34": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.35": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.36": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.37": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.38": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.39": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.4": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.40": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.41": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.42": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.43": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.44": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.45": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.46": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.47": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.48": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.49": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.5": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.50": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.51": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.52": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.53": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.54": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.55": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.56": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.57": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.58": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.59": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.6": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.60": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.61": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.62": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.63": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.64": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.65": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.66": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.67": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.68": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.69": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.7": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.70": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.71": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.72": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.73": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.74": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.75": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.76": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.77": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.78": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.79": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.8": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.80": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.81": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.82": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.83": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.84": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.85": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.86": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.87": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.88": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.89": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.9": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.90": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.91": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.92": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.93": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.94": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.95": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.96": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.97": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.98": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.down_proj_list.99": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.0": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.1": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.10": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.100": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.101": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.102": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.103": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.104": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.105": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.106": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.107": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.108": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.109": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.11": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.110": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.111": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.112": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.113": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.114": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.115": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.116": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.117": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.118": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.119": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.12": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.120": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.121": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.122": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.123": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.124": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.125": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.126": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.127": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.128": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.129": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.13": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.130": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.131": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.132": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.133": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.134": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.135": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.136": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.137": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.138": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.139": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.14": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.140": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.141": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.142": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.143": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.144": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.145": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.146": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.147": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.148": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.149": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.15": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.150": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.151": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.152": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.153": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.154": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.155": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.156": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.157": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.158": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.159": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.16": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.160": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.161": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.162": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.163": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.164": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.165": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.166": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.167": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.168": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.169": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.17": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.170": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.171": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.172": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.173": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.174": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.175": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.176": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.177": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.178": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.179": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.18": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.180": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.181": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.182": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.183": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.184": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.185": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.186": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.187": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.188": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.189": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.19": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.190": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.191": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.192": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.193": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.194": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.195": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.196": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.197": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.198": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.199": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.2": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.20": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.200": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.201": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.202": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.203": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.204": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.205": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.206": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.207": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.208": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.209": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.21": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.210": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.211": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.212": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.213": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.214": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.215": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.216": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.217": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.218": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.219": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.22": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.220": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.221": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.222": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.223": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.224": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.225": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.226": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.227": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.228": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.229": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.23": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.230": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.231": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.232": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.233": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.234": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.235": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.236": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.237": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.238": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.239": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.24": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.240": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.241": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.242": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.243": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.244": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.245": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.246": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.247": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.248": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.249": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.25": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.250": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.251": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.252": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.253": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.254": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.255": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.26": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.27": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.28": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.29": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.3": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.30": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.31": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.32": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.33": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.34": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.35": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.36": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.37": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.38": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.39": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.4": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.40": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.41": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.42": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.43": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.44": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.45": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.46": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.47": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.48": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.49": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.5": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.50": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.51": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.52": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.53": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.54": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.55": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.56": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.57": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.58": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.59": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.6": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.60": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.61": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.62": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.63": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.64": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.65": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.66": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.67": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.68": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.69": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.7": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.70": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.71": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.72": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.73": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.74": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.75": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.76": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.77": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.78": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.79": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.8": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.80": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.81": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.82": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.83": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.84": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.85": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.86": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.87": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.88": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.89": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.9": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.90": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.91": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.92": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.93": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.94": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.95": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.96": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.97": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.98": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.experts.gate_up_proj_list.99": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.shared_expert.down_proj": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.shared_expert.gate_proj": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 512, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.shared_expert.up_proj": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 512, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.mlp.shared_expert_gate": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16, + "out_features": 1, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.self_attn.k_proj": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 512, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.self_attn.o_proj": { + "group_size": 128, + "in_features": 4096, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 65536, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 4096, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.self_attn.q_proj": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 131072, + "out_features": 8192, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.39.self_attn.v_proj": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 512, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.linear_attn.in_proj_a": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 512, + "out_features": 32, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.linear_attn.in_proj_b": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 512, + "out_features": 32, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.linear_attn.in_proj_qkv": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 131072, + "out_features": 8192, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.linear_attn.in_proj_z": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 65536, + "out_features": 4096, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.linear_attn.out_proj": { + "group_size": 128, + "in_features": 4096, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 65536, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 4096, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.0": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.1": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.10": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.100": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.101": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.102": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.103": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.104": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.105": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.106": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.107": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.108": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.109": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.11": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.110": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.111": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.112": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.113": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.114": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.115": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.116": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.117": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.118": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.119": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.12": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.120": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.121": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.122": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.123": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.124": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.125": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.126": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.127": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.128": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.129": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.13": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.130": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.131": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.132": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.133": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.134": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.135": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.136": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.137": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.138": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.139": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.14": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.140": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.141": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.142": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.143": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.144": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.145": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.146": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.147": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.148": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.149": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.15": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.150": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.151": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.152": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.153": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.154": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.155": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.156": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.157": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.158": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.159": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.16": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.160": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.161": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.162": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.163": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.164": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.165": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.166": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.167": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.168": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.169": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.17": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.170": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.171": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.172": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.173": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.174": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.175": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.176": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.177": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.178": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.179": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.18": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.180": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.181": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.182": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.183": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.184": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.185": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.186": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.187": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.188": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.189": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.19": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.190": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.191": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.192": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.193": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.194": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.195": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.196": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.197": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.198": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.199": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.2": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.20": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.200": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.201": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.202": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.203": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.204": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.205": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.206": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.207": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.208": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.209": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.21": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.210": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.211": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.212": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.213": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.214": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.215": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.216": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.217": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.218": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.219": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.22": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.220": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.221": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.222": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.223": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.224": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.225": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.226": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.227": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.228": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.229": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.23": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.230": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.231": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.232": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.233": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.234": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.235": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.236": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.237": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.238": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.239": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.24": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.240": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.241": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.242": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.243": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.244": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.245": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.246": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.247": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.248": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.249": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.25": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.250": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.251": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.252": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.253": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.254": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.255": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.26": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.27": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.28": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.29": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.3": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.30": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.31": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.32": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.33": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.34": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.35": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.36": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.37": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.38": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.39": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.4": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.40": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.41": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.42": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.43": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.44": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.45": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.46": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.47": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.48": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.49": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.5": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.50": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.51": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.52": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.53": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.54": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.55": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.56": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.57": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.58": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.59": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.6": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.60": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.61": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.62": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.63": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.64": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.65": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.66": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.67": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.68": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.69": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.7": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.70": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.71": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.72": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.73": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.74": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.75": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.76": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.77": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.78": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.79": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.8": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.80": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.81": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.82": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.83": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.84": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.85": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.86": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.87": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.88": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.89": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.9": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.90": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.91": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.92": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.93": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.94": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.95": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.96": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.97": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.98": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.down_proj_list.99": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.0": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.1": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.10": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.100": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.101": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.102": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.103": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.104": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.105": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.106": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.107": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.108": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.109": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.11": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.110": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.111": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.112": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.113": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.114": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.115": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.116": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.117": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.118": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.119": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.12": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.120": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.121": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.122": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.123": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.124": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.125": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.126": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.127": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.128": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.129": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.13": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.130": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.131": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.132": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.133": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.134": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.135": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.136": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.137": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.138": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.139": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.14": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.140": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.141": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.142": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.143": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.144": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.145": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.146": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.147": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.148": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.149": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.15": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.150": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.151": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.152": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.153": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.154": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.155": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.156": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.157": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.158": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.159": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.16": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.160": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.161": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.162": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.163": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.164": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.165": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.166": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.167": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.168": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.169": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.17": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.170": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.171": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.172": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.173": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.174": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.175": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.176": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.177": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.178": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.179": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.18": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.180": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.181": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.182": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.183": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.184": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.185": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.186": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.187": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.188": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.189": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.19": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.190": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.191": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.192": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.193": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.194": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.195": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.196": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.197": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.198": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.199": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.2": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.20": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.200": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.201": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.202": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.203": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.204": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.205": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.206": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.207": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.208": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.209": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.21": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.210": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.211": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.212": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.213": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.214": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.215": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.216": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.217": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.218": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.219": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.22": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.220": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.221": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.222": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.223": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.224": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.225": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.226": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.227": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.228": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.229": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.23": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.230": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.231": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.232": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.233": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.234": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.235": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.236": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.237": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.238": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.239": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.24": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.240": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.241": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.242": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.243": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.244": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.245": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.246": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.247": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.248": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.249": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.25": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.250": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.251": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.252": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.253": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.254": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.255": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.26": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.27": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.28": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.29": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.3": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.30": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.31": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.32": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.33": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.34": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.35": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.36": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.37": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.38": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.39": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.4": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.40": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.41": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.42": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.43": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.44": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.45": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.46": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.47": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.48": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.49": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.5": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.50": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.51": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.52": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.53": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.54": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.55": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.56": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.57": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.58": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.59": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.6": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.60": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.61": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.62": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.63": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.64": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.65": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.66": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.67": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.68": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.69": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.7": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.70": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.71": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.72": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.73": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.74": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.75": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.76": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.77": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.78": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.79": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.8": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.80": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.81": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.82": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.83": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.84": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.85": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.86": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.87": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.88": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.89": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.9": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.90": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.91": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.92": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.93": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.94": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.95": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.96": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.97": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.98": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.experts.gate_up_proj_list.99": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.shared_expert.down_proj": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.shared_expert.gate_proj": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 512, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.shared_expert.up_proj": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 512, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.4.mlp.shared_expert_gate": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16, + "out_features": 1, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.linear_attn.in_proj_a": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 512, + "out_features": 32, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.linear_attn.in_proj_b": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 512, + "out_features": 32, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.linear_attn.in_proj_qkv": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 131072, + "out_features": 8192, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.linear_attn.in_proj_z": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 65536, + "out_features": 4096, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.linear_attn.out_proj": { + "group_size": 128, + "in_features": 4096, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 65536, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 4096, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.0": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.1": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.10": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.100": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.101": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.102": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.103": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.104": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.105": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.106": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.107": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.108": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.109": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.11": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.110": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.111": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.112": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.113": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.114": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.115": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.116": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.117": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.118": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.119": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.12": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.120": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.121": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.122": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.123": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.124": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.125": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.126": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.127": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.128": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.129": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.13": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.130": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.131": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.132": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.133": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.134": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.135": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.136": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.137": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.138": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.139": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.14": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.140": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.141": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.142": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.143": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.144": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.145": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.146": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.147": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.148": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.149": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.15": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.150": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.151": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.152": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.153": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.154": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.155": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.156": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.157": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.158": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.159": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.16": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.160": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.161": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.162": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.163": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.164": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.165": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.166": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.167": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.168": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.169": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.17": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.170": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.171": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.172": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.173": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.174": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.175": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.176": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.177": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.178": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.179": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.18": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.180": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.181": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.182": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.183": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.184": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.185": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.186": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.187": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.188": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.189": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.19": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.190": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.191": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.192": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.193": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.194": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.195": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.196": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.197": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.198": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.199": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.2": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.20": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.200": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.201": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.202": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.203": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.204": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.205": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.206": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.207": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.208": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.209": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.21": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.210": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.211": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.212": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.213": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.214": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.215": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.216": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.217": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.218": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.219": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.22": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.220": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.221": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.222": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.223": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.224": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.225": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.226": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.227": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.228": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.229": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.23": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.230": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.231": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.232": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.233": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.234": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.235": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.236": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.237": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.238": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.239": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.24": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.240": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.241": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.242": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.243": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.244": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.245": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.246": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.247": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.248": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.249": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.25": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.250": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.251": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.252": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.253": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.254": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.255": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.26": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.27": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.28": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.29": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.3": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.30": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.31": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.32": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.33": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.34": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.35": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.36": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.37": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.38": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.39": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.4": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.40": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.41": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.42": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.43": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.44": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.45": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.46": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.47": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.48": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.49": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.5": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.50": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.51": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.52": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.53": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.54": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.55": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.56": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.57": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.58": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.59": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.6": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.60": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.61": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.62": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.63": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.64": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.65": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.66": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.67": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.68": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.69": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.7": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.70": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.71": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.72": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.73": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.74": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.75": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.76": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.77": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.78": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.79": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.8": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.80": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.81": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.82": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.83": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.84": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.85": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.86": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.87": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.88": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.89": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.9": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.90": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.91": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.92": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.93": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.94": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.95": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.96": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.97": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.98": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.down_proj_list.99": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.0": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.1": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.10": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.100": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.101": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.102": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.103": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.104": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.105": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.106": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.107": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.108": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.109": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.11": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.110": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.111": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.112": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.113": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.114": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.115": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.116": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.117": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.118": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.119": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.12": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.120": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.121": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.122": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.123": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.124": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.125": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.126": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.127": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.128": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.129": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.13": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.130": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.131": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.132": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.133": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.134": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.135": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.136": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.137": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.138": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.139": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.14": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.140": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.141": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.142": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.143": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.144": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.145": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.146": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.147": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.148": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.149": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.15": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.150": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.151": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.152": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.153": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.154": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.155": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.156": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.157": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.158": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.159": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.16": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.160": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.161": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.162": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.163": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.164": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.165": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.166": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.167": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.168": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.169": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.17": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.170": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.171": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.172": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.173": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.174": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.175": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.176": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.177": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.178": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.179": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.18": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.180": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.181": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.182": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.183": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.184": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.185": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.186": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.187": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.188": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.189": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.19": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.190": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.191": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.192": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.193": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.194": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.195": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.196": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.197": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.198": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.199": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.2": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.20": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.200": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.201": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.202": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.203": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.204": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.205": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.206": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.207": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.208": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.209": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.21": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.210": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.211": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.212": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.213": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.214": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.215": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.216": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.217": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.218": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.219": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.22": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.220": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.221": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.222": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.223": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.224": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.225": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.226": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.227": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.228": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.229": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.23": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.230": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.231": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.232": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.233": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.234": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.235": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.236": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.237": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.238": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.239": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.24": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.240": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.241": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.242": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.243": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.244": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.245": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.246": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.247": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.248": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.249": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.25": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.250": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.251": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.252": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.253": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.254": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.255": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.26": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.27": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.28": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.29": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.3": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.30": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.31": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.32": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.33": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.34": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.35": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.36": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.37": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.38": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.39": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.4": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.40": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.41": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.42": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.43": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.44": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.45": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.46": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.47": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.48": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.49": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.5": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.50": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.51": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.52": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.53": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.54": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.55": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.56": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.57": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.58": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.59": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.6": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.60": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.61": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.62": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.63": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.64": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.65": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.66": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.67": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.68": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.69": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.7": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.70": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.71": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.72": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.73": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.74": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.75": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.76": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.77": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.78": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.79": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.8": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.80": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.81": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.82": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.83": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.84": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.85": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.86": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.87": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.88": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.89": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.9": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.90": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.91": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.92": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.93": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.94": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.95": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.96": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.97": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.98": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.experts.gate_up_proj_list.99": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.shared_expert.down_proj": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.shared_expert.gate_proj": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 512, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.shared_expert.up_proj": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 512, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.5.mlp.shared_expert_gate": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16, + "out_features": 1, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.linear_attn.in_proj_a": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 512, + "out_features": 32, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.linear_attn.in_proj_b": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 512, + "out_features": 32, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.linear_attn.in_proj_qkv": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 131072, + "out_features": 8192, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.linear_attn.in_proj_z": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 65536, + "out_features": 4096, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.linear_attn.out_proj": { + "group_size": 128, + "in_features": 4096, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 65536, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 4096, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.0": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.1": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.10": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.100": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.101": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.102": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.103": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.104": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.105": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.106": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.107": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.108": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.109": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.11": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.110": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.111": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.112": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.113": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.114": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.115": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.116": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.117": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.118": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.119": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.12": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.120": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.121": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.122": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.123": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.124": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.125": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.126": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.127": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.128": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.129": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.13": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.130": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.131": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.132": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.133": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.134": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.135": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.136": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.137": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.138": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.139": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.14": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.140": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.141": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.142": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.143": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.144": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.145": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.146": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.147": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.148": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.149": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.15": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.150": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.151": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.152": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.153": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.154": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.155": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.156": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.157": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.158": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.159": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.16": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.160": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.161": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.162": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.163": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.164": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.165": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.166": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.167": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.168": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.169": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.17": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.170": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.171": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.172": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.173": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.174": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.175": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.176": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.177": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.178": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.179": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.18": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.180": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.181": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.182": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.183": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.184": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.185": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.186": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.187": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.188": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.189": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.19": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.190": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.191": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.192": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.193": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.194": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.195": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.196": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.197": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.198": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.199": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.2": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.20": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.200": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.201": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.202": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.203": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.204": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.205": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.206": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.207": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.208": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.209": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.21": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.210": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.211": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.212": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.213": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.214": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.215": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.216": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.217": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.218": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.219": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.22": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.220": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.221": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.222": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.223": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.224": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.225": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.226": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.227": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.228": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.229": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.23": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.230": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.231": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.232": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.233": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.234": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.235": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.236": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.237": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.238": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.239": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.24": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.240": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.241": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.242": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.243": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.244": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.245": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.246": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.247": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.248": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.249": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.25": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.250": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.251": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.252": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.253": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.254": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.255": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.26": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.27": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.28": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.29": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.3": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.30": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.31": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.32": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.33": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.34": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.35": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.36": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.37": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.38": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.39": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.4": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.40": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.41": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.42": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.43": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.44": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.45": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.46": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.47": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.48": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.49": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.5": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.50": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.51": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.52": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.53": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.54": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.55": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.56": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.57": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.58": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.59": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.6": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.60": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.61": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.62": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.63": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.64": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.65": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.66": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.67": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.68": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.69": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.7": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.70": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.71": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.72": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.73": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.74": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.75": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.76": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.77": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.78": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.79": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.8": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.80": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.81": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.82": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.83": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.84": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.85": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.86": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.87": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.88": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.89": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.9": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.90": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.91": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.92": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.93": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.94": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.95": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.96": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.97": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.98": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.down_proj_list.99": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.0": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.1": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.10": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.100": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.101": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.102": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.103": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.104": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.105": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.106": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.107": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.108": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.109": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.11": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.110": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.111": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.112": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.113": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.114": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.115": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.116": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.117": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.118": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.119": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.12": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.120": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.121": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.122": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.123": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.124": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.125": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.126": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.127": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.128": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.129": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.13": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.130": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.131": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.132": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.133": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.134": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.135": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.136": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.137": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.138": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.139": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.14": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.140": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.141": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.142": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.143": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.144": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.145": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.146": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.147": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.148": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.149": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.15": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.150": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.151": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.152": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.153": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.154": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.155": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.156": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.157": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.158": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.159": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.16": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.160": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.161": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.162": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.163": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.164": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.165": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.166": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.167": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.168": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.169": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.17": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.170": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.171": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.172": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.173": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.174": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.175": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.176": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.177": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.178": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.179": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.18": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.180": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.181": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.182": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.183": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.184": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.185": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.186": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.187": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.188": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.189": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.19": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.190": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.191": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.192": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.193": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.194": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.195": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.196": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.197": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.198": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.199": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.2": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.20": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.200": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.201": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.202": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.203": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.204": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.205": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.206": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.207": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.208": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.209": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.21": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.210": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.211": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.212": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.213": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.214": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.215": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.216": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.217": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.218": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.219": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.22": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.220": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.221": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.222": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.223": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.224": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.225": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.226": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.227": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.228": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.229": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.23": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.230": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.231": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.232": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.233": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.234": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.235": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.236": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.237": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.238": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.239": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.24": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.240": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.241": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.242": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.243": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.244": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.245": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.246": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.247": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.248": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.249": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.25": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.250": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.251": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.252": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.253": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.254": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.255": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.26": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.27": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.28": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.29": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.3": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.30": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.31": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.32": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.33": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.34": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.35": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.36": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.37": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.38": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.39": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.4": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.40": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.41": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.42": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.43": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.44": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.45": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.46": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.47": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.48": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.49": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.5": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.50": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.51": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.52": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.53": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.54": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.55": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.56": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.57": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.58": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.59": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.6": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.60": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.61": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.62": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.63": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.64": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.65": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.66": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.67": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.68": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.69": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.7": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.70": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.71": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.72": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.73": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.74": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.75": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.76": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.77": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.78": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.79": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.8": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.80": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.81": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.82": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.83": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.84": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.85": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.86": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.87": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.88": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.89": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.9": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.90": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.91": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.92": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.93": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.94": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.95": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.96": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.97": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.98": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.experts.gate_up_proj_list.99": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.shared_expert.down_proj": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.shared_expert.gate_proj": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 512, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.shared_expert.up_proj": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 512, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.6.mlp.shared_expert_gate": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16, + "out_features": 1, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.0": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.1": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.10": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.100": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.101": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.102": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.103": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.104": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.105": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.106": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.107": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.108": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.109": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.11": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.110": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.111": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.112": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.113": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.114": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.115": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.116": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.117": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.118": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.119": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.12": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.120": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.121": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.122": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.123": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.124": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.125": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.126": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.127": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.128": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.129": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.13": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.130": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.131": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.132": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.133": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.134": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.135": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.136": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.137": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.138": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.139": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.14": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.140": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.141": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.142": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.143": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.144": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.145": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.146": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.147": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.148": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.149": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.15": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.150": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.151": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.152": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.153": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.154": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.155": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.156": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.157": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.158": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.159": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.16": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.160": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.161": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.162": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.163": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.164": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.165": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.166": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.167": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.168": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.169": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.17": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.170": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.171": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.172": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.173": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.174": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.175": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.176": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.177": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.178": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.179": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.18": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.180": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.181": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.182": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.183": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.184": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.185": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.186": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.187": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.188": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.189": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.19": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.190": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.191": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.192": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.193": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.194": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.195": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.196": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.197": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.198": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.199": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.2": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.20": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.200": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.201": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.202": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.203": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.204": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.205": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.206": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.207": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.208": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.209": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.21": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.210": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.211": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.212": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.213": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.214": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.215": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.216": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.217": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.218": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.219": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.22": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.220": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.221": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.222": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.223": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.224": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.225": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.226": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.227": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.228": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.229": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.23": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.230": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.231": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.232": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.233": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.234": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.235": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.236": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.237": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.238": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.239": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.24": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.240": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.241": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.242": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.243": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.244": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.245": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.246": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.247": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.248": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.249": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.25": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.250": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.251": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.252": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.253": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.254": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.255": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.26": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.27": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.28": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.29": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.3": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.30": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.31": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.32": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.33": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.34": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.35": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.36": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.37": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.38": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.39": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.4": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.40": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.41": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.42": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.43": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.44": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.45": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.46": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.47": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.48": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.49": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.5": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.50": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.51": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.52": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.53": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.54": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.55": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.56": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.57": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.58": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.59": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.6": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.60": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.61": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.62": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.63": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.64": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.65": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.66": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.67": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.68": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.69": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.7": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.70": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.71": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.72": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.73": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.74": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.75": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.76": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.77": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.78": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.79": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.8": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.80": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.81": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.82": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.83": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.84": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.85": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.86": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.87": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.88": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.89": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.9": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.90": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.91": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.92": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.93": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.94": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.95": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.96": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.97": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.98": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.down_proj_list.99": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.0": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.1": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.10": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.100": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.101": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.102": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.103": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.104": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.105": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.106": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.107": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.108": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.109": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.11": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.110": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.111": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.112": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.113": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.114": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.115": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.116": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.117": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.118": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.119": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.12": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.120": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.121": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.122": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.123": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.124": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.125": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.126": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.127": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.128": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.129": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.13": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.130": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.131": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.132": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.133": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.134": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.135": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.136": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.137": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.138": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.139": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.14": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.140": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.141": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.142": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.143": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.144": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.145": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.146": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.147": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.148": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.149": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.15": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.150": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.151": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.152": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.153": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.154": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.155": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.156": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.157": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.158": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.159": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.16": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.160": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.161": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.162": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.163": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.164": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.165": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.166": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.167": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.168": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.169": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.17": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.170": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.171": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.172": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.173": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.174": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.175": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.176": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.177": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.178": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.179": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.18": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.180": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.181": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.182": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.183": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.184": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.185": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.186": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.187": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.188": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.189": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.19": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.190": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.191": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.192": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.193": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.194": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.195": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.196": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.197": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.198": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.199": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.2": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.20": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.200": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.201": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.202": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.203": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.204": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.205": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.206": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.207": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.208": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.209": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.21": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.210": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.211": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.212": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.213": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.214": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.215": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.216": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.217": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.218": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.219": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.22": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.220": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.221": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.222": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.223": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.224": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.225": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.226": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.227": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.228": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.229": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.23": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.230": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.231": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.232": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.233": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.234": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.235": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.236": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.237": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.238": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.239": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.24": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.240": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.241": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.242": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.243": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.244": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.245": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.246": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.247": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.248": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.249": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.25": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.250": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.251": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.252": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.253": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.254": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.255": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.26": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.27": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.28": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.29": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.3": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.30": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.31": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.32": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.33": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.34": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.35": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.36": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.37": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.38": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.39": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.4": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.40": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.41": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.42": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.43": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.44": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.45": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.46": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.47": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.48": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.49": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.5": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.50": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.51": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.52": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.53": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.54": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.55": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.56": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.57": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.58": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.59": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.6": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.60": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.61": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.62": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.63": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.64": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.65": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.66": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.67": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.68": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.69": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.7": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.70": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.71": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.72": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.73": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.74": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.75": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.76": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.77": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.78": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.79": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.8": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.80": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.81": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.82": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.83": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.84": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.85": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.86": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.87": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.88": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.89": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.9": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.90": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.91": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.92": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.93": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.94": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.95": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.96": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.97": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.98": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.experts.gate_up_proj_list.99": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.shared_expert.down_proj": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.shared_expert.gate_proj": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 512, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.shared_expert.up_proj": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 512, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.mlp.shared_expert_gate": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16, + "out_features": 1, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.self_attn.k_proj": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 512, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.self_attn.o_proj": { + "group_size": 128, + "in_features": 4096, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 65536, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 4096, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.self_attn.q_proj": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 131072, + "out_features": 8192, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.7.self_attn.v_proj": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 512, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.linear_attn.in_proj_a": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 512, + "out_features": 32, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.linear_attn.in_proj_b": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 512, + "out_features": 32, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.linear_attn.in_proj_qkv": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 131072, + "out_features": 8192, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.linear_attn.in_proj_z": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 65536, + "out_features": 4096, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.linear_attn.out_proj": { + "group_size": 128, + "in_features": 4096, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 65536, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 4096, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.0": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.1": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.10": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.100": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.101": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.102": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.103": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.104": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.105": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.106": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.107": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.108": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.109": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.11": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.110": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.111": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.112": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.113": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.114": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.115": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.116": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.117": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.118": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.119": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.12": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.120": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.121": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.122": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.123": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.124": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.125": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.126": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.127": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.128": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.129": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.13": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.130": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.131": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.132": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.133": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.134": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.135": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.136": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.137": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.138": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.139": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.14": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.140": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.141": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.142": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.143": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.144": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.145": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.146": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.147": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.148": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.149": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.15": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.150": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.151": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.152": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.153": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.154": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.155": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.156": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.157": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.158": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.159": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.16": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.160": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.161": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.162": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.163": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.164": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.165": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.166": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.167": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.168": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.169": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.17": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.170": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.171": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.172": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.173": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.174": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.175": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.176": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.177": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.178": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.179": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.18": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.180": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.181": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.182": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.183": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.184": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.185": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.186": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.187": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.188": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.189": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.19": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.190": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.191": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.192": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.193": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.194": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.195": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.196": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.197": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.198": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.199": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.2": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.20": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.200": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.201": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.202": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.203": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.204": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.205": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.206": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.207": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.208": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.209": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.21": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.210": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.211": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.212": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.213": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.214": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.215": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.216": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.217": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.218": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.219": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.22": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.220": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.221": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.222": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.223": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.224": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.225": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.226": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.227": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.228": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.229": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.23": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.230": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.231": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.232": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.233": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.234": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.235": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.236": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.237": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.238": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.239": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.24": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.240": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.241": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.242": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.243": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.244": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.245": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.246": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.247": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.248": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.249": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.25": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.250": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.251": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.252": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.253": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.254": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.255": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.26": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.27": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.28": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.29": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.3": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.30": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.31": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.32": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.33": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.34": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.35": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.36": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.37": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.38": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.39": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.4": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.40": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.41": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.42": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.43": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.44": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.45": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.46": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.47": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.48": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.49": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.5": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.50": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.51": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.52": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.53": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.54": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.55": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.56": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.57": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.58": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.59": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.6": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.60": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.61": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.62": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.63": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.64": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.65": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.66": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.67": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.68": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.69": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.7": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.70": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.71": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.72": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.73": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.74": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.75": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.76": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.77": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.78": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.79": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.8": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.80": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.81": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.82": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.83": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.84": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.85": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.86": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.87": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.88": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.89": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.9": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.90": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.91": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.92": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.93": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.94": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.95": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.96": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.97": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.98": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.down_proj_list.99": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.0": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.1": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.10": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.100": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.101": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.102": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.103": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.104": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.105": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.106": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.107": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.108": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.109": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.11": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.110": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.111": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.112": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.113": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.114": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.115": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.116": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.117": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.118": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.119": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.12": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.120": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.121": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.122": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.123": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.124": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.125": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.126": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.127": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.128": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.129": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.13": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.130": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.131": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.132": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.133": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.134": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.135": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.136": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.137": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.138": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.139": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.14": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.140": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.141": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.142": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.143": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.144": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.145": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.146": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.147": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.148": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.149": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.15": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.150": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.151": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.152": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.153": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.154": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.155": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.156": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.157": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.158": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.159": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.16": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.160": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.161": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.162": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.163": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.164": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.165": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.166": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.167": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.168": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.169": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.17": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.170": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.171": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.172": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.173": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.174": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.175": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.176": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.177": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.178": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.179": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.18": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.180": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.181": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.182": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.183": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.184": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.185": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.186": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.187": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.188": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.189": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.19": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.190": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.191": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.192": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.193": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.194": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.195": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.196": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.197": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.198": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.199": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.2": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.20": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.200": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.201": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.202": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.203": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.204": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.205": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.206": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.207": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.208": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.209": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.21": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.210": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.211": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.212": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.213": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.214": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.215": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.216": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.217": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.218": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.219": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.22": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.220": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.221": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.222": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.223": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.224": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.225": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.226": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.227": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.228": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.229": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.23": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.230": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.231": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.232": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.233": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.234": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.235": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.236": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.237": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.238": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.239": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.24": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.240": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.241": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.242": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.243": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.244": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.245": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.246": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.247": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.248": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.249": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.25": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.250": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.251": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.252": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.253": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.254": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.255": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.26": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.27": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.28": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.29": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.3": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.30": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.31": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.32": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.33": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.34": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.35": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.36": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.37": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.38": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.39": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.4": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.40": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.41": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.42": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.43": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.44": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.45": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.46": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.47": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.48": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.49": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.5": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.50": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.51": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.52": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.53": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.54": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.55": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.56": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.57": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.58": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.59": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.6": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.60": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.61": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.62": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.63": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.64": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.65": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.66": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.67": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.68": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.69": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.7": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.70": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.71": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.72": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.73": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.74": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.75": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.76": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.77": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.78": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.79": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.8": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.80": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.81": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.82": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.83": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.84": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.85": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.86": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.87": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.88": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.89": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.9": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.90": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.91": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.92": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.93": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.94": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.95": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.96": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.97": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.98": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.experts.gate_up_proj_list.99": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.shared_expert.down_proj": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.shared_expert.gate_proj": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 512, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.shared_expert.up_proj": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 512, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.8.mlp.shared_expert_gate": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16, + "out_features": 1, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.linear_attn.in_proj_a": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 512, + "out_features": 32, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.linear_attn.in_proj_b": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 512, + "out_features": 32, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.linear_attn.in_proj_qkv": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 131072, + "out_features": 8192, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.linear_attn.in_proj_z": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 65536, + "out_features": 4096, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.linear_attn.out_proj": { + "group_size": 128, + "in_features": 4096, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 65536, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 4096, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.0": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.1": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.10": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.100": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.101": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.102": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.103": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.104": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.105": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.106": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.107": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.108": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.109": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.11": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.110": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.111": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.112": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.113": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.114": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.115": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.116": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.117": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.118": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.119": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.12": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.120": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.121": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.122": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.123": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.124": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.125": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.126": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.127": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.128": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.129": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.13": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.130": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.131": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.132": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.133": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.134": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.135": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.136": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.137": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.138": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.139": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.14": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.140": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.141": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.142": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.143": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.144": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.145": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.146": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.147": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.148": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.149": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.15": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.150": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.151": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.152": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.153": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.154": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.155": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.156": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.157": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.158": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.159": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.16": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.160": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.161": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.162": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.163": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.164": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.165": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.166": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.167": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.168": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.169": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.17": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.170": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.171": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.172": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.173": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.174": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.175": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.176": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.177": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.178": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.179": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.18": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.180": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.181": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.182": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.183": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.184": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.185": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.186": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.187": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.188": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.189": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.19": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.190": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.191": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.192": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.193": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.194": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.195": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.196": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.197": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.198": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.199": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.2": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.20": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.200": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.201": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.202": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.203": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.204": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.205": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.206": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.207": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.208": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.209": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.21": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.210": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.211": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.212": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.213": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.214": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.215": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.216": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.217": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.218": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.219": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.22": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.220": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.221": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.222": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.223": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.224": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.225": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.226": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.227": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.228": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.229": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.23": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.230": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.231": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.232": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.233": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.234": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.235": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.236": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.237": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.238": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.239": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.24": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.240": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.241": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.242": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.243": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.244": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.245": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.246": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.247": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.248": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.249": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.25": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.250": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.251": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.252": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.253": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.254": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.255": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.26": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.27": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.28": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.29": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.3": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.30": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.31": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.32": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.33": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.34": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.35": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.36": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.37": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.38": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.39": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.4": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.40": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.41": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.42": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.43": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.44": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.45": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.46": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.47": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.48": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.49": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.5": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.50": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.51": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.52": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.53": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.54": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.55": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.56": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.57": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.58": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.59": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.6": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.60": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.61": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.62": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.63": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.64": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.65": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.66": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.67": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.68": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.69": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.7": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.70": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.71": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.72": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.73": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.74": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.75": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.76": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.77": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.78": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.79": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.8": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.80": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.81": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.82": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.83": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.84": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.85": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.86": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.87": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.88": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.89": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.9": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.90": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.91": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.92": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.93": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.94": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.95": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.96": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.97": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.98": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.down_proj_list.99": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.0": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.1": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.10": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.100": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.101": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.102": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.103": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.104": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.105": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.106": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.107": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.108": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.109": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.11": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.110": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.111": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.112": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.113": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.114": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.115": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.116": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.117": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.118": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.119": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.12": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.120": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.121": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.122": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.123": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.124": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.125": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.126": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.127": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.128": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.129": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.13": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.130": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.131": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.132": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.133": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.134": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.135": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.136": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.137": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.138": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.139": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.14": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.140": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.141": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.142": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.143": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.144": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.145": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.146": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.147": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.148": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.149": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.15": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.150": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.151": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.152": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.153": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.154": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.155": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.156": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.157": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.158": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.159": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.16": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.160": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.161": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.162": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.163": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.164": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.165": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.166": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.167": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.168": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.169": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.17": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.170": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.171": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.172": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.173": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.174": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.175": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.176": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.177": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.178": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.179": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.18": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.180": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.181": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.182": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.183": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.184": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.185": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.186": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.187": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.188": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.189": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.19": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.190": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.191": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.192": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.193": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.194": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.195": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.196": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.197": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.198": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.199": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.2": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.20": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.200": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.201": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.202": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.203": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.204": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.205": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.206": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.207": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.208": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.209": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.21": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.210": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.211": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.212": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.213": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.214": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.215": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.216": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.217": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.218": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.219": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.22": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.220": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.221": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.222": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.223": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.224": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.225": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.226": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.227": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.228": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.229": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.23": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.230": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.231": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.232": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.233": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.234": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.235": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.236": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.237": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.238": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.239": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.24": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.240": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.241": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.242": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.243": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.244": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.245": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.246": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.247": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.248": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.249": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.25": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.250": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.251": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.252": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.253": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.254": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.255": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.26": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.27": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.28": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.29": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.3": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.30": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.31": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.32": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.33": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.34": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.35": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.36": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.37": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.38": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.39": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.4": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.40": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.41": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.42": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.43": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.44": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.45": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.46": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.47": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.48": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.49": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.5": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.50": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.51": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.52": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.53": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.54": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.55": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.56": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.57": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.58": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.59": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.6": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.60": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.61": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.62": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.63": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.64": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.65": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.66": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.67": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.68": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.69": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.7": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.70": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.71": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.72": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.73": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.74": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.75": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.76": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.77": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.78": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.79": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.8": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.80": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.81": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.82": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.83": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.84": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.85": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.86": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.87": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.88": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.89": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.9": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.90": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.91": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.92": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.93": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.94": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.95": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.96": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.97": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.98": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.experts.gate_up_proj_list.99": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16384, + "out_features": 1024, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.shared_expert.down_proj": { + "group_size": 128, + "in_features": 512, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 2048, + "packing": "int3_packed_u32", + "quant_in_features": 512, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.shared_expert.gate_proj": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 512, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.shared_expert.up_proj": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 8192, + "out_features": 512, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + }, + "model.language_model.layers.9.mlp.shared_expert_gate": { + "group_size": 128, + "in_features": 2048, + "linear_dtype": "bfloat16", + "nbits": 3, + "num_groups": 16, + "out_features": 1, + "packing": "int3_packed_u32", + "quant_in_features": 2048, + "runtime_backend": "torch", + "scale_zero_dtype": "float16" + } + }, + "results": { + "Method": "dashq", + "Model": "Qwen/Qwen3.5-35B-A3B", + "ModelSizeGB": 17.479980288, + "OriginalSizeGB": 71.903878016, + "PPL": 7.142297744750977, + "Params": "{'bits': 3, 'group_size': 128, 'scale_zero_dtype': 'float16', 'n_samples': 128, 'moe_hessian_scope': 'shared', 'use_error_compensation': True, 'use_optimal_shrinkage': True, 'use_weighted_quantization': True, 'symmetric': False, 'low_memory_optimization': False}", + "QuantTime": 2075.001234292984, + "arc_challenge": 60.238907849829346, + "arc_easy": 81.27104377104376, + "commonsense_qa": 84.43898443898445, + "hellaswag": 80.47201752638917, + "lambada_openai": 69.377061905686, + "openbookqa": 43.2, + "piqa": 82.2633297062024, + "truthfulqa_mc2": 55.1748786130417, + "winogrande": 73.4017363851618, + "zeroshot_avg": 69.98199557737095 + } +} \ No newline at end of file