Hugging Face's logo

jinreiyu
/
Bonsai-REDUX2-MLX-edge-d8-Lo6

Text Generation
MLX
Safetensors
qwen3
ternary
1.58-bit
mlx-lm
mlx-swift
apple-silicon
on-device
prismml
bonsai
conversational
2-bit
Model card Files
xet
 Community
jinreiyu commited on
Commit
8908cc2
·
verified ·
1 Parent(s): 5d50845

Upload 13 files

Browse files
Files changed (7) hide show
  1. TRIAL_INFO.txt +16 -9
  2. config.json +2581 -87
  3. generation_config.json +9 -0
  4. model.safetensors +2 -2
  5. model.safetensors.index.json +1 -1
  6. tokenizer.json +2 -2
  7. tokenizer_config.json +6 -0
TRIAL_INFO.txt CHANGED
@@ -1,9 +1,16 @@
1
- What this is: GEN4 pure 2-bit ternary MLX (the 'magic 2 GB' target)
2
- Generation: GEN4 (gen 4)
3
- Trial: Trial 36 of GEN4 (auto-picked, lowest score)
4
- Refusals / KL: 3 / 100, KL=0.0000 (vs GEN4 parent)
5
- Source HF: /Volumes/ARKsafe/PerplexityInstalls/ABBLITERATIONFOLDER/gen4/work/gen4-best
6
- Parent: /Volumes/ARKsafe/PerplexityInstalls/ABBLITERATIONFOLDER/gen4/work/parent (symlink to previous gen's best)
7
- Quant: pure ternary 2-bit (scale, -scale, 0; group 128)
8
- Caveat: pure-2-bit ternarization rounds away most of the LoRA delta.
9
- For the FULL abliteration strength, see the FP16 source.
 
 
 
 
 
 
 
 
1
+ What this is: redux2 EDGE variant -- max abliteration preservation under 4 GB
2
+ Trial: Best Pareto trial from redux2 (50-trial Heretic + projection_pass_v2)
3
+ Quant scheme:
4
+ tier1 (8-bit affine, group 64) : mlp.down_proj on ALL 36 layers
5
+ (residual-stream writer, heavily modified
6
+ by projection_pass_v2)
7
+ tier2 (6-bit affine, group 64) : self_attn.o_proj on LATE layers (12-35) only
8
+ (the layers projection_pass_v2 actually
9
+ touched -- their weights drifted enough
10
+ from ternary to tolerate affine quant)
11
+ rest : 2-bit ternary (zero-preserving custom scheme)
12
+ Why "Lo6": "Late-layer o_proj at 6-bit". Restricting to layers 12-35
13
+ avoids the redux1-mixed-do6 failure mode where ALL 36
14
+ attn.o_proj at 6-bit broke the model (early layers are
15
+ too-close-to-pure-ternary; 6-bit affine has no zero level).
16
+ Config: rope_theta=1000000.0, rope_scaling=yarn (MLX-LM compatible)
config.json CHANGED
@@ -1,27 +1,17 @@
1
  {
2
- "vocab_size": 151669,
3
- "max_position_embeddings": 65536,
4
- "hidden_size": 4096,
5
- "intermediate_size": 12288,
6
- "num_hidden_layers": 36,
7
- "num_attention_heads": 32,
8
- "use_sliding_window": false,
9
- "sliding_window": null,
10
- "max_window_layers": 28,
11
- "num_key_value_heads": 8,
12
  "head_dim": 128,
13
  "hidden_act": "silu",
 
14
  "initializer_range": 0.02,
15
- "rms_norm_eps": 1e-06,
16
- "use_cache": true,
17
- "rope_theta": 1000000.0,
18
- "rope_scaling": {
19
- "rope_type": "yarn",
20
- "factor": 4.0,
21
- "original_max_position_embeddings": 16384
22
- },
23
- "attention_bias": false,
24
- "attention_dropout": 0.0,
25
  "layer_types": [
26
  "full_attention",
27
  "full_attention",
@@ -60,76 +50,2580 @@
60
  "full_attention",
61
  "full_attention"
62
  ],
63
- "return_dict": true,
64
- "output_hidden_states": false,
65
- "torchscript": false,
66
- "dtype": null,
67
- "pruned_heads": {},
68
- "tie_word_embeddings": false,
69
- "chunk_size_feed_forward": 0,
70
- "is_encoder_decoder": false,
71
- "is_decoder": false,
72
- "cross_attention_hidden_size": null,
73
- "add_cross_attention": false,
74
- "tie_encoder_decoder": false,
75
- "architectures": [
76
- "Qwen3ForCausalLM"
77
- ],
78
- "finetuning_task": null,
79
- "id2label": {
80
- "0": "LABEL_0",
81
- "1": "LABEL_1"
82
  },
83
- "label2id": {
84
- "LABEL_0": 0,
85
- "LABEL_1": 1
 
 
 
 
 
 
 
 
86
  },
87
- "task_specific_params": null,
88
- "problem_type": null,
89
- "tokenizer_class": null,
90
- "prefix": null,
91
- "bos_token_id": null,
92
- "pad_token_id": 151643,
93
- "eos_token_id": 151645,
94
- "sep_token_id": null,
95
- "decoder_start_token_id": null,
96
- "max_length": 20,
97
- "min_length": 0,
98
- "do_sample": false,
99
- "early_stopping": false,
100
- "num_beams": 1,
101
- "temperature": 1.0,
102
- "top_k": 50,
103
- "top_p": 1.0,
104
- "typical_p": 1.0,
105
- "repetition_penalty": 1.0,
106
- "length_penalty": 1.0,
107
- "no_repeat_ngram_size": 0,
108
- "encoder_no_repeat_ngram_size": 0,
109
- "bad_words_ids": null,
110
- "num_return_sequences": 1,
111
- "output_scores": false,
112
- "return_dict_in_generate": false,
113
- "forced_bos_token_id": null,
114
- "forced_eos_token_id": null,
115
- "remove_invalid_values": false,
116
- "exponential_decay_length_penalty": null,
117
- "suppress_tokens": null,
118
- "begin_suppress_tokens": [
119
- 151643,
120
- 151645
121
- ],
122
- "num_beam_groups": 1,
123
- "diversity_penalty": 0.0,
124
- "_name_or_path": "",
125
- "transformers_version": "4.57.6",
126
- "no_bias": true,
127
- "tf_legacy_loss": false,
128
- "use_bfloat16": false,
129
- "model_type": "qwen3",
130
- "output_attentions": false,
131
  "quantization": {
132
  "group_size": 128,
133
- "bits": 2
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
134
  }
135
  }
 
1
  {
2
+ "architectures": [
3
+ "Qwen3ForCausalLM"
4
+ ],
5
+ "attention_bias": false,
6
+ "attention_dropout": 0.0,
7
+ "bos_token_id": null,
8
+ "dtype": "float16",
9
+ "eos_token_id": 151645,
 
 
10
  "head_dim": 128,
11
  "hidden_act": "silu",
12
+ "hidden_size": 4096,
13
  "initializer_range": 0.02,
14
+ "intermediate_size": 12288,
 
 
 
 
 
 
 
 
 
15
  "layer_types": [
16
  "full_attention",
17
  "full_attention",
 
50
  "full_attention",
51
  "full_attention"
52
  ],
53
+ "max_position_embeddings": 65536,
54
+ "max_window_layers": 28,
55
+ "model_type": "qwen3",
56
+ "num_attention_heads": 32,
57
+ "num_hidden_layers": 36,
58
+ "num_key_value_heads": 8,
59
+ "pad_token_id": 151643,
60
+ "rms_norm_eps": 1e-06,
61
+ "rope_parameters": {
62
+ "factor": 4.0,
63
+ "original_max_position_embeddings": 16384,
64
+ "rope_theta": 1000000.0,
65
+ "rope_type": "yarn"
 
 
 
 
 
 
66
  },
67
+ "sliding_window": null,
68
+ "tie_word_embeddings": false,
69
+ "transformers_version": "5.7.0",
70
+ "use_cache": true,
71
+ "use_sliding_window": false,
72
+ "vocab_size": 151669,
73
+ "rope_theta": 1000000.0,
74
+ "rope_scaling": {
75
+ "rope_type": "yarn",
76
+ "factor": 4.0,
77
+ "original_max_position_embeddings": 16384
78
  },
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
79
  "quantization": {
80
  "group_size": 128,
81
+ "bits": 2,
82
+ "mode": "affine",
83
+ "lm_head": {
84
+ "group_size": 128,
85
+ "bits": 2,
86
+ "mode": "affine"
87
+ },
88
+ "model.embed_tokens": {
89
+ "group_size": 128,
90
+ "bits": 2,
91
+ "mode": "affine"
92
+ },
93
+ "model.layers.0.mlp.down_proj": {
94
+ "group_size": 64,
95
+ "bits": 8,
96
+ "mode": "affine"
97
+ },
98
+ "model.layers.0.mlp.gate_proj": {
99
+ "group_size": 128,
100
+ "bits": 2,
101
+ "mode": "affine"
102
+ },
103
+ "model.layers.0.mlp.up_proj": {
104
+ "group_size": 128,
105
+ "bits": 2,
106
+ "mode": "affine"
107
+ },
108
+ "model.layers.0.self_attn.k_proj": {
109
+ "group_size": 128,
110
+ "bits": 2,
111
+ "mode": "affine"
112
+ },
113
+ "model.layers.0.self_attn.o_proj": {
114
+ "group_size": 128,
115
+ "bits": 2,
116
+ "mode": "affine"
117
+ },
118
+ "model.layers.0.self_attn.q_proj": {
119
+ "group_size": 128,
120
+ "bits": 2,
121
+ "mode": "affine"
122
+ },
123
+ "model.layers.0.self_attn.v_proj": {
124
+ "group_size": 128,
125
+ "bits": 2,
126
+ "mode": "affine"
127
+ },
128
+ "model.layers.1.mlp.down_proj": {
129
+ "group_size": 64,
130
+ "bits": 8,
131
+ "mode": "affine"
132
+ },
133
+ "model.layers.1.mlp.gate_proj": {
134
+ "group_size": 128,
135
+ "bits": 2,
136
+ "mode": "affine"
137
+ },
138
+ "model.layers.1.mlp.up_proj": {
139
+ "group_size": 128,
140
+ "bits": 2,
141
+ "mode": "affine"
142
+ },
143
+ "model.layers.1.self_attn.k_proj": {
144
+ "group_size": 128,
145
+ "bits": 2,
146
+ "mode": "affine"
147
+ },
148
+ "model.layers.1.self_attn.o_proj": {
149
+ "group_size": 128,
150
+ "bits": 2,
151
+ "mode": "affine"
152
+ },
153
+ "model.layers.1.self_attn.q_proj": {
154
+ "group_size": 128,
155
+ "bits": 2,
156
+ "mode": "affine"
157
+ },
158
+ "model.layers.1.self_attn.v_proj": {
159
+ "group_size": 128,
160
+ "bits": 2,
161
+ "mode": "affine"
162
+ },
163
+ "model.layers.2.mlp.down_proj": {
164
+ "group_size": 64,
165
+ "bits": 8,
166
+ "mode": "affine"
167
+ },
168
+ "model.layers.2.mlp.gate_proj": {
169
+ "group_size": 128,
170
+ "bits": 2,
171
+ "mode": "affine"
172
+ },
173
+ "model.layers.2.mlp.up_proj": {
174
+ "group_size": 128,
175
+ "bits": 2,
176
+ "mode": "affine"
177
+ },
178
+ "model.layers.2.self_attn.k_proj": {
179
+ "group_size": 128,
180
+ "bits": 2,
181
+ "mode": "affine"
182
+ },
183
+ "model.layers.2.self_attn.o_proj": {
184
+ "group_size": 128,
185
+ "bits": 2,
186
+ "mode": "affine"
187
+ },
188
+ "model.layers.2.self_attn.q_proj": {
189
+ "group_size": 128,
190
+ "bits": 2,
191
+ "mode": "affine"
192
+ },
193
+ "model.layers.2.self_attn.v_proj": {
194
+ "group_size": 128,
195
+ "bits": 2,
196
+ "mode": "affine"
197
+ },
198
+ "model.layers.3.mlp.down_proj": {
199
+ "group_size": 64,
200
+ "bits": 8,
201
+ "mode": "affine"
202
+ },
203
+ "model.layers.3.mlp.gate_proj": {
204
+ "group_size": 128,
205
+ "bits": 2,
206
+ "mode": "affine"
207
+ },
208
+ "model.layers.3.mlp.up_proj": {
209
+ "group_size": 128,
210
+ "bits": 2,
211
+ "mode": "affine"
212
+ },
213
+ "model.layers.3.self_attn.k_proj": {
214
+ "group_size": 128,
215
+ "bits": 2,
216
+ "mode": "affine"
217
+ },
218
+ "model.layers.3.self_attn.o_proj": {
219
+ "group_size": 128,
220
+ "bits": 2,
221
+ "mode": "affine"
222
+ },
223
+ "model.layers.10.mlp.down_proj": {
224
+ "group_size": 64,
225
+ "bits": 8,
226
+ "mode": "affine"
227
+ },
228
+ "model.layers.10.mlp.gate_proj": {
229
+ "group_size": 128,
230
+ "bits": 2,
231
+ "mode": "affine"
232
+ },
233
+ "model.layers.10.mlp.up_proj": {
234
+ "group_size": 128,
235
+ "bits": 2,
236
+ "mode": "affine"
237
+ },
238
+ "model.layers.10.self_attn.k_proj": {
239
+ "group_size": 128,
240
+ "bits": 2,
241
+ "mode": "affine"
242
+ },
243
+ "model.layers.10.self_attn.o_proj": {
244
+ "group_size": 128,
245
+ "bits": 2,
246
+ "mode": "affine"
247
+ },
248
+ "model.layers.10.self_attn.q_proj": {
249
+ "group_size": 128,
250
+ "bits": 2,
251
+ "mode": "affine"
252
+ },
253
+ "model.layers.10.self_attn.v_proj": {
254
+ "group_size": 128,
255
+ "bits": 2,
256
+ "mode": "affine"
257
+ },
258
+ "model.layers.11.mlp.down_proj": {
259
+ "group_size": 64,
260
+ "bits": 8,
261
+ "mode": "affine"
262
+ },
263
+ "model.layers.11.mlp.gate_proj": {
264
+ "group_size": 128,
265
+ "bits": 2,
266
+ "mode": "affine"
267
+ },
268
+ "model.layers.11.mlp.up_proj": {
269
+ "group_size": 128,
270
+ "bits": 2,
271
+ "mode": "affine"
272
+ },
273
+ "model.layers.11.self_attn.k_proj": {
274
+ "group_size": 128,
275
+ "bits": 2,
276
+ "mode": "affine"
277
+ },
278
+ "model.layers.11.self_attn.o_proj": {
279
+ "group_size": 128,
280
+ "bits": 2,
281
+ "mode": "affine"
282
+ },
283
+ "model.layers.11.self_attn.q_proj": {
284
+ "group_size": 128,
285
+ "bits": 2,
286
+ "mode": "affine"
287
+ },
288
+ "model.layers.11.self_attn.v_proj": {
289
+ "group_size": 128,
290
+ "bits": 2,
291
+ "mode": "affine"
292
+ },
293
+ "model.layers.12.mlp.down_proj": {
294
+ "group_size": 64,
295
+ "bits": 8,
296
+ "mode": "affine"
297
+ },
298
+ "model.layers.12.mlp.gate_proj": {
299
+ "group_size": 128,
300
+ "bits": 2,
301
+ "mode": "affine"
302
+ },
303
+ "model.layers.12.mlp.up_proj": {
304
+ "group_size": 128,
305
+ "bits": 2,
306
+ "mode": "affine"
307
+ },
308
+ "model.layers.12.self_attn.k_proj": {
309
+ "group_size": 128,
310
+ "bits": 2,
311
+ "mode": "affine"
312
+ },
313
+ "model.layers.12.self_attn.o_proj": {
314
+ "group_size": 64,
315
+ "bits": 6,
316
+ "mode": "affine"
317
+ },
318
+ "model.layers.12.self_attn.q_proj": {
319
+ "group_size": 128,
320
+ "bits": 2,
321
+ "mode": "affine"
322
+ },
323
+ "model.layers.12.self_attn.v_proj": {
324
+ "group_size": 128,
325
+ "bits": 2,
326
+ "mode": "affine"
327
+ },
328
+ "model.layers.13.mlp.down_proj": {
329
+ "group_size": 64,
330
+ "bits": 8,
331
+ "mode": "affine"
332
+ },
333
+ "model.layers.13.mlp.gate_proj": {
334
+ "group_size": 128,
335
+ "bits": 2,
336
+ "mode": "affine"
337
+ },
338
+ "model.layers.13.mlp.up_proj": {
339
+ "group_size": 128,
340
+ "bits": 2,
341
+ "mode": "affine"
342
+ },
343
+ "model.layers.13.self_attn.k_proj": {
344
+ "group_size": 128,
345
+ "bits": 2,
346
+ "mode": "affine"
347
+ },
348
+ "model.layers.13.self_attn.o_proj": {
349
+ "group_size": 64,
350
+ "bits": 6,
351
+ "mode": "affine"
352
+ },
353
+ "model.layers.13.self_attn.q_proj": {
354
+ "group_size": 128,
355
+ "bits": 2,
356
+ "mode": "affine"
357
+ },
358
+ "model.layers.13.self_attn.v_proj": {
359
+ "group_size": 128,
360
+ "bits": 2,
361
+ "mode": "affine"
362
+ },
363
+ "model.layers.3.self_attn.q_proj": {
364
+ "group_size": 128,
365
+ "bits": 2,
366
+ "mode": "affine"
367
+ },
368
+ "model.layers.3.self_attn.v_proj": {
369
+ "group_size": 128,
370
+ "bits": 2,
371
+ "mode": "affine"
372
+ },
373
+ "model.layers.4.mlp.down_proj": {
374
+ "group_size": 64,
375
+ "bits": 8,
376
+ "mode": "affine"
377
+ },
378
+ "model.layers.4.mlp.gate_proj": {
379
+ "group_size": 128,
380
+ "bits": 2,
381
+ "mode": "affine"
382
+ },
383
+ "model.layers.4.mlp.up_proj": {
384
+ "group_size": 128,
385
+ "bits": 2,
386
+ "mode": "affine"
387
+ },
388
+ "model.layers.4.self_attn.k_proj": {
389
+ "group_size": 128,
390
+ "bits": 2,
391
+ "mode": "affine"
392
+ },
393
+ "model.layers.4.self_attn.o_proj": {
394
+ "group_size": 128,
395
+ "bits": 2,
396
+ "mode": "affine"
397
+ },
398
+ "model.layers.4.self_attn.q_proj": {
399
+ "group_size": 128,
400
+ "bits": 2,
401
+ "mode": "affine"
402
+ },
403
+ "model.layers.4.self_attn.v_proj": {
404
+ "group_size": 128,
405
+ "bits": 2,
406
+ "mode": "affine"
407
+ },
408
+ "model.layers.5.mlp.down_proj": {
409
+ "group_size": 64,
410
+ "bits": 8,
411
+ "mode": "affine"
412
+ },
413
+ "model.layers.5.mlp.gate_proj": {
414
+ "group_size": 128,
415
+ "bits": 2,
416
+ "mode": "affine"
417
+ },
418
+ "model.layers.5.mlp.up_proj": {
419
+ "group_size": 128,
420
+ "bits": 2,
421
+ "mode": "affine"
422
+ },
423
+ "model.layers.5.self_attn.k_proj": {
424
+ "group_size": 128,
425
+ "bits": 2,
426
+ "mode": "affine"
427
+ },
428
+ "model.layers.5.self_attn.o_proj": {
429
+ "group_size": 128,
430
+ "bits": 2,
431
+ "mode": "affine"
432
+ },
433
+ "model.layers.5.self_attn.q_proj": {
434
+ "group_size": 128,
435
+ "bits": 2,
436
+ "mode": "affine"
437
+ },
438
+ "model.layers.5.self_attn.v_proj": {
439
+ "group_size": 128,
440
+ "bits": 2,
441
+ "mode": "affine"
442
+ },
443
+ "model.layers.6.mlp.down_proj": {
444
+ "group_size": 64,
445
+ "bits": 8,
446
+ "mode": "affine"
447
+ },
448
+ "model.layers.6.mlp.gate_proj": {
449
+ "group_size": 128,
450
+ "bits": 2,
451
+ "mode": "affine"
452
+ },
453
+ "model.layers.6.mlp.up_proj": {
454
+ "group_size": 128,
455
+ "bits": 2,
456
+ "mode": "affine"
457
+ },
458
+ "model.layers.6.self_attn.k_proj": {
459
+ "group_size": 128,
460
+ "bits": 2,
461
+ "mode": "affine"
462
+ },
463
+ "model.layers.6.self_attn.o_proj": {
464
+ "group_size": 128,
465
+ "bits": 2,
466
+ "mode": "affine"
467
+ },
468
+ "model.layers.6.self_attn.q_proj": {
469
+ "group_size": 128,
470
+ "bits": 2,
471
+ "mode": "affine"
472
+ },
473
+ "model.layers.6.self_attn.v_proj": {
474
+ "group_size": 128,
475
+ "bits": 2,
476
+ "mode": "affine"
477
+ },
478
+ "model.layers.7.mlp.down_proj": {
479
+ "group_size": 64,
480
+ "bits": 8,
481
+ "mode": "affine"
482
+ },
483
+ "model.layers.7.mlp.gate_proj": {
484
+ "group_size": 128,
485
+ "bits": 2,
486
+ "mode": "affine"
487
+ },
488
+ "model.layers.7.mlp.up_proj": {
489
+ "group_size": 128,
490
+ "bits": 2,
491
+ "mode": "affine"
492
+ },
493
+ "model.layers.7.self_attn.k_proj": {
494
+ "group_size": 128,
495
+ "bits": 2,
496
+ "mode": "affine"
497
+ },
498
+ "model.layers.7.self_attn.o_proj": {
499
+ "group_size": 128,
500
+ "bits": 2,
501
+ "mode": "affine"
502
+ },
503
+ "model.layers.7.self_attn.q_proj": {
504
+ "group_size": 128,
505
+ "bits": 2,
506
+ "mode": "affine"
507
+ },
508
+ "model.layers.7.self_attn.v_proj": {
509
+ "group_size": 128,
510
+ "bits": 2,
511
+ "mode": "affine"
512
+ },
513
+ "model.layers.8.mlp.down_proj": {
514
+ "group_size": 64,
515
+ "bits": 8,
516
+ "mode": "affine"
517
+ },
518
+ "model.layers.8.mlp.gate_proj": {
519
+ "group_size": 128,
520
+ "bits": 2,
521
+ "mode": "affine"
522
+ },
523
+ "model.layers.8.mlp.up_proj": {
524
+ "group_size": 128,
525
+ "bits": 2,
526
+ "mode": "affine"
527
+ },
528
+ "model.layers.8.self_attn.k_proj": {
529
+ "group_size": 128,
530
+ "bits": 2,
531
+ "mode": "affine"
532
+ },
533
+ "model.layers.8.self_attn.o_proj": {
534
+ "group_size": 128,
535
+ "bits": 2,
536
+ "mode": "affine"
537
+ },
538
+ "model.layers.8.self_attn.q_proj": {
539
+ "group_size": 128,
540
+ "bits": 2,
541
+ "mode": "affine"
542
+ },
543
+ "model.layers.8.self_attn.v_proj": {
544
+ "group_size": 128,
545
+ "bits": 2,
546
+ "mode": "affine"
547
+ },
548
+ "model.layers.9.mlp.down_proj": {
549
+ "group_size": 64,
550
+ "bits": 8,
551
+ "mode": "affine"
552
+ },
553
+ "model.layers.9.mlp.gate_proj": {
554
+ "group_size": 128,
555
+ "bits": 2,
556
+ "mode": "affine"
557
+ },
558
+ "model.layers.9.mlp.up_proj": {
559
+ "group_size": 128,
560
+ "bits": 2,
561
+ "mode": "affine"
562
+ },
563
+ "model.layers.9.self_attn.k_proj": {
564
+ "group_size": 128,
565
+ "bits": 2,
566
+ "mode": "affine"
567
+ },
568
+ "model.layers.9.self_attn.o_proj": {
569
+ "group_size": 128,
570
+ "bits": 2,
571
+ "mode": "affine"
572
+ },
573
+ "model.layers.9.self_attn.q_proj": {
574
+ "group_size": 128,
575
+ "bits": 2,
576
+ "mode": "affine"
577
+ },
578
+ "model.layers.9.self_attn.v_proj": {
579
+ "group_size": 128,
580
+ "bits": 2,
581
+ "mode": "affine"
582
+ },
583
+ "model.layers.14.mlp.down_proj": {
584
+ "group_size": 64,
585
+ "bits": 8,
586
+ "mode": "affine"
587
+ },
588
+ "model.layers.14.mlp.gate_proj": {
589
+ "group_size": 128,
590
+ "bits": 2,
591
+ "mode": "affine"
592
+ },
593
+ "model.layers.14.mlp.up_proj": {
594
+ "group_size": 128,
595
+ "bits": 2,
596
+ "mode": "affine"
597
+ },
598
+ "model.layers.14.self_attn.k_proj": {
599
+ "group_size": 128,
600
+ "bits": 2,
601
+ "mode": "affine"
602
+ },
603
+ "model.layers.14.self_attn.o_proj": {
604
+ "group_size": 64,
605
+ "bits": 6,
606
+ "mode": "affine"
607
+ },
608
+ "model.layers.14.self_attn.q_proj": {
609
+ "group_size": 128,
610
+ "bits": 2,
611
+ "mode": "affine"
612
+ },
613
+ "model.layers.14.self_attn.v_proj": {
614
+ "group_size": 128,
615
+ "bits": 2,
616
+ "mode": "affine"
617
+ },
618
+ "model.layers.15.mlp.down_proj": {
619
+ "group_size": 64,
620
+ "bits": 8,
621
+ "mode": "affine"
622
+ },
623
+ "model.layers.15.mlp.gate_proj": {
624
+ "group_size": 128,
625
+ "bits": 2,
626
+ "mode": "affine"
627
+ },
628
+ "model.layers.15.mlp.up_proj": {
629
+ "group_size": 128,
630
+ "bits": 2,
631
+ "mode": "affine"
632
+ },
633
+ "model.layers.15.self_attn.k_proj": {
634
+ "group_size": 128,
635
+ "bits": 2,
636
+ "mode": "affine"
637
+ },
638
+ "model.layers.15.self_attn.o_proj": {
639
+ "group_size": 64,
640
+ "bits": 6,
641
+ "mode": "affine"
642
+ },
643
+ "model.layers.15.self_attn.q_proj": {
644
+ "group_size": 128,
645
+ "bits": 2,
646
+ "mode": "affine"
647
+ },
648
+ "model.layers.15.self_attn.v_proj": {
649
+ "group_size": 128,
650
+ "bits": 2,
651
+ "mode": "affine"
652
+ },
653
+ "model.layers.16.mlp.down_proj": {
654
+ "group_size": 64,
655
+ "bits": 8,
656
+ "mode": "affine"
657
+ },
658
+ "model.layers.16.mlp.gate_proj": {
659
+ "group_size": 128,
660
+ "bits": 2,
661
+ "mode": "affine"
662
+ },
663
+ "model.layers.16.mlp.up_proj": {
664
+ "group_size": 128,
665
+ "bits": 2,
666
+ "mode": "affine"
667
+ },
668
+ "model.layers.16.self_attn.k_proj": {
669
+ "group_size": 128,
670
+ "bits": 2,
671
+ "mode": "affine"
672
+ },
673
+ "model.layers.16.self_attn.o_proj": {
674
+ "group_size": 64,
675
+ "bits": 6,
676
+ "mode": "affine"
677
+ },
678
+ "model.layers.16.self_attn.q_proj": {
679
+ "group_size": 128,
680
+ "bits": 2,
681
+ "mode": "affine"
682
+ },
683
+ "model.layers.16.self_attn.v_proj": {
684
+ "group_size": 128,
685
+ "bits": 2,
686
+ "mode": "affine"
687
+ },
688
+ "model.layers.17.mlp.down_proj": {
689
+ "group_size": 64,
690
+ "bits": 8,
691
+ "mode": "affine"
692
+ },
693
+ "model.layers.17.mlp.gate_proj": {
694
+ "group_size": 128,
695
+ "bits": 2,
696
+ "mode": "affine"
697
+ },
698
+ "model.layers.17.mlp.up_proj": {
699
+ "group_size": 128,
700
+ "bits": 2,
701
+ "mode": "affine"
702
+ },
703
+ "model.layers.17.self_attn.k_proj": {
704
+ "group_size": 128,
705
+ "bits": 2,
706
+ "mode": "affine"
707
+ },
708
+ "model.layers.17.self_attn.o_proj": {
709
+ "group_size": 64,
710
+ "bits": 6,
711
+ "mode": "affine"
712
+ },
713
+ "model.layers.17.self_attn.q_proj": {
714
+ "group_size": 128,
715
+ "bits": 2,
716
+ "mode": "affine"
717
+ },
718
+ "model.layers.17.self_attn.v_proj": {
719
+ "group_size": 128,
720
+ "bits": 2,
721
+ "mode": "affine"
722
+ },
723
+ "model.layers.18.mlp.down_proj": {
724
+ "group_size": 64,
725
+ "bits": 8,
726
+ "mode": "affine"
727
+ },
728
+ "model.layers.18.mlp.gate_proj": {
729
+ "group_size": 128,
730
+ "bits": 2,
731
+ "mode": "affine"
732
+ },
733
+ "model.layers.18.mlp.up_proj": {
734
+ "group_size": 128,
735
+ "bits": 2,
736
+ "mode": "affine"
737
+ },
738
+ "model.layers.18.self_attn.k_proj": {
739
+ "group_size": 128,
740
+ "bits": 2,
741
+ "mode": "affine"
742
+ },
743
+ "model.layers.18.self_attn.o_proj": {
744
+ "group_size": 64,
745
+ "bits": 6,
746
+ "mode": "affine"
747
+ },
748
+ "model.layers.18.self_attn.q_proj": {
749
+ "group_size": 128,
750
+ "bits": 2,
751
+ "mode": "affine"
752
+ },
753
+ "model.layers.18.self_attn.v_proj": {
754
+ "group_size": 128,
755
+ "bits": 2,
756
+ "mode": "affine"
757
+ },
758
+ "model.layers.19.mlp.down_proj": {
759
+ "group_size": 64,
760
+ "bits": 8,
761
+ "mode": "affine"
762
+ },
763
+ "model.layers.19.mlp.gate_proj": {
764
+ "group_size": 128,
765
+ "bits": 2,
766
+ "mode": "affine"
767
+ },
768
+ "model.layers.19.mlp.up_proj": {
769
+ "group_size": 128,
770
+ "bits": 2,
771
+ "mode": "affine"
772
+ },
773
+ "model.layers.19.self_attn.k_proj": {
774
+ "group_size": 128,
775
+ "bits": 2,
776
+ "mode": "affine"
777
+ },
778
+ "model.layers.19.self_attn.o_proj": {
779
+ "group_size": 64,
780
+ "bits": 6,
781
+ "mode": "affine"
782
+ },
783
+ "model.layers.19.self_attn.q_proj": {
784
+ "group_size": 128,
785
+ "bits": 2,
786
+ "mode": "affine"
787
+ },
788
+ "model.layers.19.self_attn.v_proj": {
789
+ "group_size": 128,
790
+ "bits": 2,
791
+ "mode": "affine"
792
+ },
793
+ "model.layers.20.mlp.down_proj": {
794
+ "group_size": 64,
795
+ "bits": 8,
796
+ "mode": "affine"
797
+ },
798
+ "model.layers.20.mlp.gate_proj": {
799
+ "group_size": 128,
800
+ "bits": 2,
801
+ "mode": "affine"
802
+ },
803
+ "model.layers.20.mlp.up_proj": {
804
+ "group_size": 128,
805
+ "bits": 2,
806
+ "mode": "affine"
807
+ },
808
+ "model.layers.20.self_attn.k_proj": {
809
+ "group_size": 128,
810
+ "bits": 2,
811
+ "mode": "affine"
812
+ },
813
+ "model.layers.20.self_attn.o_proj": {
814
+ "group_size": 64,
815
+ "bits": 6,
816
+ "mode": "affine"
817
+ },
818
+ "model.layers.20.self_attn.q_proj": {
819
+ "group_size": 128,
820
+ "bits": 2,
821
+ "mode": "affine"
822
+ },
823
+ "model.layers.20.self_attn.v_proj": {
824
+ "group_size": 128,
825
+ "bits": 2,
826
+ "mode": "affine"
827
+ },
828
+ "model.layers.21.mlp.down_proj": {
829
+ "group_size": 64,
830
+ "bits": 8,
831
+ "mode": "affine"
832
+ },
833
+ "model.layers.21.mlp.gate_proj": {
834
+ "group_size": 128,
835
+ "bits": 2,
836
+ "mode": "affine"
837
+ },
838
+ "model.layers.21.mlp.up_proj": {
839
+ "group_size": 128,
840
+ "bits": 2,
841
+ "mode": "affine"
842
+ },
843
+ "model.layers.21.self_attn.k_proj": {
844
+ "group_size": 128,
845
+ "bits": 2,
846
+ "mode": "affine"
847
+ },
848
+ "model.layers.21.self_attn.o_proj": {
849
+ "group_size": 64,
850
+ "bits": 6,
851
+ "mode": "affine"
852
+ },
853
+ "model.layers.21.self_attn.q_proj": {
854
+ "group_size": 128,
855
+ "bits": 2,
856
+ "mode": "affine"
857
+ },
858
+ "model.layers.21.self_attn.v_proj": {
859
+ "group_size": 128,
860
+ "bits": 2,
861
+ "mode": "affine"
862
+ },
863
+ "model.layers.22.mlp.down_proj": {
864
+ "group_size": 64,
865
+ "bits": 8,
866
+ "mode": "affine"
867
+ },
868
+ "model.layers.22.mlp.gate_proj": {
869
+ "group_size": 128,
870
+ "bits": 2,
871
+ "mode": "affine"
872
+ },
873
+ "model.layers.22.mlp.up_proj": {
874
+ "group_size": 128,
875
+ "bits": 2,
876
+ "mode": "affine"
877
+ },
878
+ "model.layers.22.self_attn.k_proj": {
879
+ "group_size": 128,
880
+ "bits": 2,
881
+ "mode": "affine"
882
+ },
883
+ "model.layers.22.self_attn.o_proj": {
884
+ "group_size": 64,
885
+ "bits": 6,
886
+ "mode": "affine"
887
+ },
888
+ "model.layers.22.self_attn.q_proj": {
889
+ "group_size": 128,
890
+ "bits": 2,
891
+ "mode": "affine"
892
+ },
893
+ "model.layers.22.self_attn.v_proj": {
894
+ "group_size": 128,
895
+ "bits": 2,
896
+ "mode": "affine"
897
+ },
898
+ "model.layers.23.mlp.down_proj": {
899
+ "group_size": 64,
900
+ "bits": 8,
901
+ "mode": "affine"
902
+ },
903
+ "model.layers.23.mlp.gate_proj": {
904
+ "group_size": 128,
905
+ "bits": 2,
906
+ "mode": "affine"
907
+ },
908
+ "model.layers.23.mlp.up_proj": {
909
+ "group_size": 128,
910
+ "bits": 2,
911
+ "mode": "affine"
912
+ },
913
+ "model.layers.23.self_attn.k_proj": {
914
+ "group_size": 128,
915
+ "bits": 2,
916
+ "mode": "affine"
917
+ },
918
+ "model.layers.23.self_attn.o_proj": {
919
+ "group_size": 64,
920
+ "bits": 6,
921
+ "mode": "affine"
922
+ },
923
+ "model.layers.23.self_attn.q_proj": {
924
+ "group_size": 128,
925
+ "bits": 2,
926
+ "mode": "affine"
927
+ },
928
+ "model.layers.23.self_attn.v_proj": {
929
+ "group_size": 128,
930
+ "bits": 2,
931
+ "mode": "affine"
932
+ },
933
+ "model.layers.24.mlp.down_proj": {
934
+ "group_size": 64,
935
+ "bits": 8,
936
+ "mode": "affine"
937
+ },
938
+ "model.layers.24.mlp.gate_proj": {
939
+ "group_size": 128,
940
+ "bits": 2,
941
+ "mode": "affine"
942
+ },
943
+ "model.layers.24.mlp.up_proj": {
944
+ "group_size": 128,
945
+ "bits": 2,
946
+ "mode": "affine"
947
+ },
948
+ "model.layers.24.self_attn.k_proj": {
949
+ "group_size": 128,
950
+ "bits": 2,
951
+ "mode": "affine"
952
+ },
953
+ "model.layers.24.self_attn.o_proj": {
954
+ "group_size": 64,
955
+ "bits": 6,
956
+ "mode": "affine"
957
+ },
958
+ "model.layers.24.self_attn.q_proj": {
959
+ "group_size": 128,
960
+ "bits": 2,
961
+ "mode": "affine"
962
+ },
963
+ "model.layers.24.self_attn.v_proj": {
964
+ "group_size": 128,
965
+ "bits": 2,
966
+ "mode": "affine"
967
+ },
968
+ "model.layers.25.mlp.down_proj": {
969
+ "group_size": 64,
970
+ "bits": 8,
971
+ "mode": "affine"
972
+ },
973
+ "model.layers.25.mlp.gate_proj": {
974
+ "group_size": 128,
975
+ "bits": 2,
976
+ "mode": "affine"
977
+ },
978
+ "model.layers.25.mlp.up_proj": {
979
+ "group_size": 128,
980
+ "bits": 2,
981
+ "mode": "affine"
982
+ },
983
+ "model.layers.25.self_attn.k_proj": {
984
+ "group_size": 128,
985
+ "bits": 2,
986
+ "mode": "affine"
987
+ },
988
+ "model.layers.25.self_attn.o_proj": {
989
+ "group_size": 64,
990
+ "bits": 6,
991
+ "mode": "affine"
992
+ },
993
+ "model.layers.25.self_attn.q_proj": {
994
+ "group_size": 128,
995
+ "bits": 2,
996
+ "mode": "affine"
997
+ },
998
+ "model.layers.25.self_attn.v_proj": {
999
+ "group_size": 128,
1000
+ "bits": 2,
1001
+ "mode": "affine"
1002
+ },
1003
+ "model.layers.26.mlp.down_proj": {
1004
+ "group_size": 64,
1005
+ "bits": 8,
1006
+ "mode": "affine"
1007
+ },
1008
+ "model.layers.26.mlp.gate_proj": {
1009
+ "group_size": 128,
1010
+ "bits": 2,
1011
+ "mode": "affine"
1012
+ },
1013
+ "model.layers.26.mlp.up_proj": {
1014
+ "group_size": 128,
1015
+ "bits": 2,
1016
+ "mode": "affine"
1017
+ },
1018
+ "model.layers.26.self_attn.k_proj": {
1019
+ "group_size": 128,
1020
+ "bits": 2,
1021
+ "mode": "affine"
1022
+ },
1023
+ "model.layers.26.self_attn.o_proj": {
1024
+ "group_size": 64,
1025
+ "bits": 6,
1026
+ "mode": "affine"
1027
+ },
1028
+ "model.layers.26.self_attn.q_proj": {
1029
+ "group_size": 128,
1030
+ "bits": 2,
1031
+ "mode": "affine"
1032
+ },
1033
+ "model.layers.26.self_attn.v_proj": {
1034
+ "group_size": 128,
1035
+ "bits": 2,
1036
+ "mode": "affine"
1037
+ },
1038
+ "model.layers.27.mlp.down_proj": {
1039
+ "group_size": 64,
1040
+ "bits": 8,
1041
+ "mode": "affine"
1042
+ },
1043
+ "model.layers.27.mlp.gate_proj": {
1044
+ "group_size": 128,
1045
+ "bits": 2,
1046
+ "mode": "affine"
1047
+ },
1048
+ "model.layers.27.mlp.up_proj": {
1049
+ "group_size": 128,
1050
+ "bits": 2,
1051
+ "mode": "affine"
1052
+ },
1053
+ "model.layers.27.self_attn.k_proj": {
1054
+ "group_size": 128,
1055
+ "bits": 2,
1056
+ "mode": "affine"
1057
+ },
1058
+ "model.layers.27.self_attn.o_proj": {
1059
+ "group_size": 64,
1060
+ "bits": 6,
1061
+ "mode": "affine"
1062
+ },
1063
+ "model.layers.27.self_attn.q_proj": {
1064
+ "group_size": 128,
1065
+ "bits": 2,
1066
+ "mode": "affine"
1067
+ },
1068
+ "model.layers.27.self_attn.v_proj": {
1069
+ "group_size": 128,
1070
+ "bits": 2,
1071
+ "mode": "affine"
1072
+ },
1073
+ "model.layers.28.mlp.down_proj": {
1074
+ "group_size": 64,
1075
+ "bits": 8,
1076
+ "mode": "affine"
1077
+ },
1078
+ "model.layers.28.mlp.gate_proj": {
1079
+ "group_size": 128,
1080
+ "bits": 2,
1081
+ "mode": "affine"
1082
+ },
1083
+ "model.layers.28.mlp.up_proj": {
1084
+ "group_size": 128,
1085
+ "bits": 2,
1086
+ "mode": "affine"
1087
+ },
1088
+ "model.layers.28.self_attn.k_proj": {
1089
+ "group_size": 128,
1090
+ "bits": 2,
1091
+ "mode": "affine"
1092
+ },
1093
+ "model.layers.28.self_attn.o_proj": {
1094
+ "group_size": 64,
1095
+ "bits": 6,
1096
+ "mode": "affine"
1097
+ },
1098
+ "model.layers.28.self_attn.q_proj": {
1099
+ "group_size": 128,
1100
+ "bits": 2,
1101
+ "mode": "affine"
1102
+ },
1103
+ "model.layers.28.self_attn.v_proj": {
1104
+ "group_size": 128,
1105
+ "bits": 2,
1106
+ "mode": "affine"
1107
+ },
1108
+ "model.layers.29.mlp.down_proj": {
1109
+ "group_size": 64,
1110
+ "bits": 8,
1111
+ "mode": "affine"
1112
+ },
1113
+ "model.layers.29.mlp.gate_proj": {
1114
+ "group_size": 128,
1115
+ "bits": 2,
1116
+ "mode": "affine"
1117
+ },
1118
+ "model.layers.29.mlp.up_proj": {
1119
+ "group_size": 128,
1120
+ "bits": 2,
1121
+ "mode": "affine"
1122
+ },
1123
+ "model.layers.29.self_attn.k_proj": {
1124
+ "group_size": 128,
1125
+ "bits": 2,
1126
+ "mode": "affine"
1127
+ },
1128
+ "model.layers.29.self_attn.o_proj": {
1129
+ "group_size": 64,
1130
+ "bits": 6,
1131
+ "mode": "affine"
1132
+ },
1133
+ "model.layers.29.self_attn.q_proj": {
1134
+ "group_size": 128,
1135
+ "bits": 2,
1136
+ "mode": "affine"
1137
+ },
1138
+ "model.layers.29.self_attn.v_proj": {
1139
+ "group_size": 128,
1140
+ "bits": 2,
1141
+ "mode": "affine"
1142
+ },
1143
+ "model.layers.30.mlp.down_proj": {
1144
+ "group_size": 64,
1145
+ "bits": 8,
1146
+ "mode": "affine"
1147
+ },
1148
+ "model.layers.30.mlp.gate_proj": {
1149
+ "group_size": 128,
1150
+ "bits": 2,
1151
+ "mode": "affine"
1152
+ },
1153
+ "model.layers.30.mlp.up_proj": {
1154
+ "group_size": 128,
1155
+ "bits": 2,
1156
+ "mode": "affine"
1157
+ },
1158
+ "model.layers.30.self_attn.k_proj": {
1159
+ "group_size": 128,
1160
+ "bits": 2,
1161
+ "mode": "affine"
1162
+ },
1163
+ "model.layers.30.self_attn.o_proj": {
1164
+ "group_size": 64,
1165
+ "bits": 6,
1166
+ "mode": "affine"
1167
+ },
1168
+ "model.layers.30.self_attn.q_proj": {
1169
+ "group_size": 128,
1170
+ "bits": 2,
1171
+ "mode": "affine"
1172
+ },
1173
+ "model.layers.30.self_attn.v_proj": {
1174
+ "group_size": 128,
1175
+ "bits": 2,
1176
+ "mode": "affine"
1177
+ },
1178
+ "model.layers.31.mlp.down_proj": {
1179
+ "group_size": 64,
1180
+ "bits": 8,
1181
+ "mode": "affine"
1182
+ },
1183
+ "model.layers.31.mlp.gate_proj": {
1184
+ "group_size": 128,
1185
+ "bits": 2,
1186
+ "mode": "affine"
1187
+ },
1188
+ "model.layers.31.mlp.up_proj": {
1189
+ "group_size": 128,
1190
+ "bits": 2,
1191
+ "mode": "affine"
1192
+ },
1193
+ "model.layers.31.self_attn.k_proj": {
1194
+ "group_size": 128,
1195
+ "bits": 2,
1196
+ "mode": "affine"
1197
+ },
1198
+ "model.layers.31.self_attn.o_proj": {
1199
+ "group_size": 64,
1200
+ "bits": 6,
1201
+ "mode": "affine"
1202
+ },
1203
+ "model.layers.31.self_attn.q_proj": {
1204
+ "group_size": 128,
1205
+ "bits": 2,
1206
+ "mode": "affine"
1207
+ },
1208
+ "model.layers.31.self_attn.v_proj": {
1209
+ "group_size": 128,
1210
+ "bits": 2,
1211
+ "mode": "affine"
1212
+ },
1213
+ "model.layers.32.mlp.down_proj": {
1214
+ "group_size": 64,
1215
+ "bits": 8,
1216
+ "mode": "affine"
1217
+ },
1218
+ "model.layers.32.mlp.gate_proj": {
1219
+ "group_size": 128,
1220
+ "bits": 2,
1221
+ "mode": "affine"
1222
+ },
1223
+ "model.layers.32.mlp.up_proj": {
1224
+ "group_size": 128,
1225
+ "bits": 2,
1226
+ "mode": "affine"
1227
+ },
1228
+ "model.layers.32.self_attn.k_proj": {
1229
+ "group_size": 128,
1230
+ "bits": 2,
1231
+ "mode": "affine"
1232
+ },
1233
+ "model.layers.32.self_attn.o_proj": {
1234
+ "group_size": 64,
1235
+ "bits": 6,
1236
+ "mode": "affine"
1237
+ },
1238
+ "model.layers.32.self_attn.q_proj": {
1239
+ "group_size": 128,
1240
+ "bits": 2,
1241
+ "mode": "affine"
1242
+ },
1243
+ "model.layers.32.self_attn.v_proj": {
1244
+ "group_size": 128,
1245
+ "bits": 2,
1246
+ "mode": "affine"
1247
+ },
1248
+ "model.layers.33.mlp.down_proj": {
1249
+ "group_size": 64,
1250
+ "bits": 8,
1251
+ "mode": "affine"
1252
+ },
1253
+ "model.layers.33.mlp.gate_proj": {
1254
+ "group_size": 128,
1255
+ "bits": 2,
1256
+ "mode": "affine"
1257
+ },
1258
+ "model.layers.33.mlp.up_proj": {
1259
+ "group_size": 128,
1260
+ "bits": 2,
1261
+ "mode": "affine"
1262
+ },
1263
+ "model.layers.33.self_attn.k_proj": {
1264
+ "group_size": 128,
1265
+ "bits": 2,
1266
+ "mode": "affine"
1267
+ },
1268
+ "model.layers.33.self_attn.o_proj": {
1269
+ "group_size": 64,
1270
+ "bits": 6,
1271
+ "mode": "affine"
1272
+ },
1273
+ "model.layers.33.self_attn.q_proj": {
1274
+ "group_size": 128,
1275
+ "bits": 2,
1276
+ "mode": "affine"
1277
+ },
1278
+ "model.layers.33.self_attn.v_proj": {
1279
+ "group_size": 128,
1280
+ "bits": 2,
1281
+ "mode": "affine"
1282
+ },
1283
+ "model.layers.34.mlp.down_proj": {
1284
+ "group_size": 64,
1285
+ "bits": 8,
1286
+ "mode": "affine"
1287
+ },
1288
+ "model.layers.34.mlp.gate_proj": {
1289
+ "group_size": 128,
1290
+ "bits": 2,
1291
+ "mode": "affine"
1292
+ },
1293
+ "model.layers.34.mlp.up_proj": {
1294
+ "group_size": 128,
1295
+ "bits": 2,
1296
+ "mode": "affine"
1297
+ },
1298
+ "model.layers.34.self_attn.k_proj": {
1299
+ "group_size": 128,
1300
+ "bits": 2,
1301
+ "mode": "affine"
1302
+ },
1303
+ "model.layers.34.self_attn.o_proj": {
1304
+ "group_size": 64,
1305
+ "bits": 6,
1306
+ "mode": "affine"
1307
+ },
1308
+ "model.layers.34.self_attn.q_proj": {
1309
+ "group_size": 128,
1310
+ "bits": 2,
1311
+ "mode": "affine"
1312
+ },
1313
+ "model.layers.34.self_attn.v_proj": {
1314
+ "group_size": 128,
1315
+ "bits": 2,
1316
+ "mode": "affine"
1317
+ },
1318
+ "model.layers.35.mlp.down_proj": {
1319
+ "group_size": 64,
1320
+ "bits": 8,
1321
+ "mode": "affine"
1322
+ },
1323
+ "model.layers.35.mlp.gate_proj": {
1324
+ "group_size": 128,
1325
+ "bits": 2,
1326
+ "mode": "affine"
1327
+ },
1328
+ "model.layers.35.mlp.up_proj": {
1329
+ "group_size": 128,
1330
+ "bits": 2,
1331
+ "mode": "affine"
1332
+ },
1333
+ "model.layers.35.self_attn.k_proj": {
1334
+ "group_size": 128,
1335
+ "bits": 2,
1336
+ "mode": "affine"
1337
+ },
1338
+ "model.layers.35.self_attn.o_proj": {
1339
+ "group_size": 64,
1340
+ "bits": 6,
1341
+ "mode": "affine"
1342
+ },
1343
+ "model.layers.35.self_attn.q_proj": {
1344
+ "group_size": 128,
1345
+ "bits": 2,
1346
+ "mode": "affine"
1347
+ },
1348
+ "model.layers.35.self_attn.v_proj": {
1349
+ "group_size": 128,
1350
+ "bits": 2,
1351
+ "mode": "affine"
1352
+ }
1353
+ },
1354
+ "quantization_config": {
1355
+ "group_size": 128,
1356
+ "bits": 2,
1357
+ "mode": "affine",
1358
+ "lm_head": {
1359
+ "group_size": 128,
1360
+ "bits": 2,
1361
+ "mode": "affine"
1362
+ },
1363
+ "model.embed_tokens": {
1364
+ "group_size": 128,
1365
+ "bits": 2,
1366
+ "mode": "affine"
1367
+ },
1368
+ "model.layers.0.mlp.down_proj": {
1369
+ "group_size": 64,
1370
+ "bits": 8,
1371
+ "mode": "affine"
1372
+ },
1373
+ "model.layers.0.mlp.gate_proj": {
1374
+ "group_size": 128,
1375
+ "bits": 2,
1376
+ "mode": "affine"
1377
+ },
1378
+ "model.layers.0.mlp.up_proj": {
1379
+ "group_size": 128,
1380
+ "bits": 2,
1381
+ "mode": "affine"
1382
+ },
1383
+ "model.layers.0.self_attn.k_proj": {
1384
+ "group_size": 128,
1385
+ "bits": 2,
1386
+ "mode": "affine"
1387
+ },
1388
+ "model.layers.0.self_attn.o_proj": {
1389
+ "group_size": 128,
1390
+ "bits": 2,
1391
+ "mode": "affine"
1392
+ },
1393
+ "model.layers.0.self_attn.q_proj": {
1394
+ "group_size": 128,
1395
+ "bits": 2,
1396
+ "mode": "affine"
1397
+ },
1398
+ "model.layers.0.self_attn.v_proj": {
1399
+ "group_size": 128,
1400
+ "bits": 2,
1401
+ "mode": "affine"
1402
+ },
1403
+ "model.layers.1.mlp.down_proj": {
1404
+ "group_size": 64,
1405
+ "bits": 8,
1406
+ "mode": "affine"
1407
+ },
1408
+ "model.layers.1.mlp.gate_proj": {
1409
+ "group_size": 128,
1410
+ "bits": 2,
1411
+ "mode": "affine"
1412
+ },
1413
+ "model.layers.1.mlp.up_proj": {
1414
+ "group_size": 128,
1415
+ "bits": 2,
1416
+ "mode": "affine"
1417
+ },
1418
+ "model.layers.1.self_attn.k_proj": {
1419
+ "group_size": 128,
1420
+ "bits": 2,
1421
+ "mode": "affine"
1422
+ },
1423
+ "model.layers.1.self_attn.o_proj": {
1424
+ "group_size": 128,
1425
+ "bits": 2,
1426
+ "mode": "affine"
1427
+ },
1428
+ "model.layers.1.self_attn.q_proj": {
1429
+ "group_size": 128,
1430
+ "bits": 2,
1431
+ "mode": "affine"
1432
+ },
1433
+ "model.layers.1.self_attn.v_proj": {
1434
+ "group_size": 128,
1435
+ "bits": 2,
1436
+ "mode": "affine"
1437
+ },
1438
+ "model.layers.2.mlp.down_proj": {
1439
+ "group_size": 64,
1440
+ "bits": 8,
1441
+ "mode": "affine"
1442
+ },
1443
+ "model.layers.2.mlp.gate_proj": {
1444
+ "group_size": 128,
1445
+ "bits": 2,
1446
+ "mode": "affine"
1447
+ },
1448
+ "model.layers.2.mlp.up_proj": {
1449
+ "group_size": 128,
1450
+ "bits": 2,
1451
+ "mode": "affine"
1452
+ },
1453
+ "model.layers.2.self_attn.k_proj": {
1454
+ "group_size": 128,
1455
+ "bits": 2,
1456
+ "mode": "affine"
1457
+ },
1458
+ "model.layers.2.self_attn.o_proj": {
1459
+ "group_size": 128,
1460
+ "bits": 2,
1461
+ "mode": "affine"
1462
+ },
1463
+ "model.layers.2.self_attn.q_proj": {
1464
+ "group_size": 128,
1465
+ "bits": 2,
1466
+ "mode": "affine"
1467
+ },
1468
+ "model.layers.2.self_attn.v_proj": {
1469
+ "group_size": 128,
1470
+ "bits": 2,
1471
+ "mode": "affine"
1472
+ },
1473
+ "model.layers.3.mlp.down_proj": {
1474
+ "group_size": 64,
1475
+ "bits": 8,
1476
+ "mode": "affine"
1477
+ },
1478
+ "model.layers.3.mlp.gate_proj": {
1479
+ "group_size": 128,
1480
+ "bits": 2,
1481
+ "mode": "affine"
1482
+ },
1483
+ "model.layers.3.mlp.up_proj": {
1484
+ "group_size": 128,
1485
+ "bits": 2,
1486
+ "mode": "affine"
1487
+ },
1488
+ "model.layers.3.self_attn.k_proj": {
1489
+ "group_size": 128,
1490
+ "bits": 2,
1491
+ "mode": "affine"
1492
+ },
1493
+ "model.layers.3.self_attn.o_proj": {
1494
+ "group_size": 128,
1495
+ "bits": 2,
1496
+ "mode": "affine"
1497
+ },
1498
+ "model.layers.10.mlp.down_proj": {
1499
+ "group_size": 64,
1500
+ "bits": 8,
1501
+ "mode": "affine"
1502
+ },
1503
+ "model.layers.10.mlp.gate_proj": {
1504
+ "group_size": 128,
1505
+ "bits": 2,
1506
+ "mode": "affine"
1507
+ },
1508
+ "model.layers.10.mlp.up_proj": {
1509
+ "group_size": 128,
1510
+ "bits": 2,
1511
+ "mode": "affine"
1512
+ },
1513
+ "model.layers.10.self_attn.k_proj": {
1514
+ "group_size": 128,
1515
+ "bits": 2,
1516
+ "mode": "affine"
1517
+ },
1518
+ "model.layers.10.self_attn.o_proj": {
1519
+ "group_size": 128,
1520
+ "bits": 2,
1521
+ "mode": "affine"
1522
+ },
1523
+ "model.layers.10.self_attn.q_proj": {
1524
+ "group_size": 128,
1525
+ "bits": 2,
1526
+ "mode": "affine"
1527
+ },
1528
+ "model.layers.10.self_attn.v_proj": {
1529
+ "group_size": 128,
1530
+ "bits": 2,
1531
+ "mode": "affine"
1532
+ },
1533
+ "model.layers.11.mlp.down_proj": {
1534
+ "group_size": 64,
1535
+ "bits": 8,
1536
+ "mode": "affine"
1537
+ },
1538
+ "model.layers.11.mlp.gate_proj": {
1539
+ "group_size": 128,
1540
+ "bits": 2,
1541
+ "mode": "affine"
1542
+ },
1543
+ "model.layers.11.mlp.up_proj": {
1544
+ "group_size": 128,
1545
+ "bits": 2,
1546
+ "mode": "affine"
1547
+ },
1548
+ "model.layers.11.self_attn.k_proj": {
1549
+ "group_size": 128,
1550
+ "bits": 2,
1551
+ "mode": "affine"
1552
+ },
1553
+ "model.layers.11.self_attn.o_proj": {
1554
+ "group_size": 128,
1555
+ "bits": 2,
1556
+ "mode": "affine"
1557
+ },
1558
+ "model.layers.11.self_attn.q_proj": {
1559
+ "group_size": 128,
1560
+ "bits": 2,
1561
+ "mode": "affine"
1562
+ },
1563
+ "model.layers.11.self_attn.v_proj": {
1564
+ "group_size": 128,
1565
+ "bits": 2,
1566
+ "mode": "affine"
1567
+ },
1568
+ "model.layers.12.mlp.down_proj": {
1569
+ "group_size": 64,
1570
+ "bits": 8,
1571
+ "mode": "affine"
1572
+ },
1573
+ "model.layers.12.mlp.gate_proj": {
1574
+ "group_size": 128,
1575
+ "bits": 2,
1576
+ "mode": "affine"
1577
+ },
1578
+ "model.layers.12.mlp.up_proj": {
1579
+ "group_size": 128,
1580
+ "bits": 2,
1581
+ "mode": "affine"
1582
+ },
1583
+ "model.layers.12.self_attn.k_proj": {
1584
+ "group_size": 128,
1585
+ "bits": 2,
1586
+ "mode": "affine"
1587
+ },
1588
+ "model.layers.12.self_attn.o_proj": {
1589
+ "group_size": 64,
1590
+ "bits": 6,
1591
+ "mode": "affine"
1592
+ },
1593
+ "model.layers.12.self_attn.q_proj": {
1594
+ "group_size": 128,
1595
+ "bits": 2,
1596
+ "mode": "affine"
1597
+ },
1598
+ "model.layers.12.self_attn.v_proj": {
1599
+ "group_size": 128,
1600
+ "bits": 2,
1601
+ "mode": "affine"
1602
+ },
1603
+ "model.layers.13.mlp.down_proj": {
1604
+ "group_size": 64,
1605
+ "bits": 8,
1606
+ "mode": "affine"
1607
+ },
1608
+ "model.layers.13.mlp.gate_proj": {
1609
+ "group_size": 128,
1610
+ "bits": 2,
1611
+ "mode": "affine"
1612
+ },
1613
+ "model.layers.13.mlp.up_proj": {
1614
+ "group_size": 128,
1615
+ "bits": 2,
1616
+ "mode": "affine"
1617
+ },
1618
+ "model.layers.13.self_attn.k_proj": {
1619
+ "group_size": 128,
1620
+ "bits": 2,
1621
+ "mode": "affine"
1622
+ },
1623
+ "model.layers.13.self_attn.o_proj": {
1624
+ "group_size": 64,
1625
+ "bits": 6,
1626
+ "mode": "affine"
1627
+ },
1628
+ "model.layers.13.self_attn.q_proj": {
1629
+ "group_size": 128,
1630
+ "bits": 2,
1631
+ "mode": "affine"
1632
+ },
1633
+ "model.layers.13.self_attn.v_proj": {
1634
+ "group_size": 128,
1635
+ "bits": 2,
1636
+ "mode": "affine"
1637
+ },
1638
+ "model.layers.3.self_attn.q_proj": {
1639
+ "group_size": 128,
1640
+ "bits": 2,
1641
+ "mode": "affine"
1642
+ },
1643
+ "model.layers.3.self_attn.v_proj": {
1644
+ "group_size": 128,
1645
+ "bits": 2,
1646
+ "mode": "affine"
1647
+ },
1648
+ "model.layers.4.mlp.down_proj": {
1649
+ "group_size": 64,
1650
+ "bits": 8,
1651
+ "mode": "affine"
1652
+ },
1653
+ "model.layers.4.mlp.gate_proj": {
1654
+ "group_size": 128,
1655
+ "bits": 2,
1656
+ "mode": "affine"
1657
+ },
1658
+ "model.layers.4.mlp.up_proj": {
1659
+ "group_size": 128,
1660
+ "bits": 2,
1661
+ "mode": "affine"
1662
+ },
1663
+ "model.layers.4.self_attn.k_proj": {
1664
+ "group_size": 128,
1665
+ "bits": 2,
1666
+ "mode": "affine"
1667
+ },
1668
+ "model.layers.4.self_attn.o_proj": {
1669
+ "group_size": 128,
1670
+ "bits": 2,
1671
+ "mode": "affine"
1672
+ },
1673
+ "model.layers.4.self_attn.q_proj": {
1674
+ "group_size": 128,
1675
+ "bits": 2,
1676
+ "mode": "affine"
1677
+ },
1678
+ "model.layers.4.self_attn.v_proj": {
1679
+ "group_size": 128,
1680
+ "bits": 2,
1681
+ "mode": "affine"
1682
+ },
1683
+ "model.layers.5.mlp.down_proj": {
1684
+ "group_size": 64,
1685
+ "bits": 8,
1686
+ "mode": "affine"
1687
+ },
1688
+ "model.layers.5.mlp.gate_proj": {
1689
+ "group_size": 128,
1690
+ "bits": 2,
1691
+ "mode": "affine"
1692
+ },
1693
+ "model.layers.5.mlp.up_proj": {
1694
+ "group_size": 128,
1695
+ "bits": 2,
1696
+ "mode": "affine"
1697
+ },
1698
+ "model.layers.5.self_attn.k_proj": {
1699
+ "group_size": 128,
1700
+ "bits": 2,
1701
+ "mode": "affine"
1702
+ },
1703
+ "model.layers.5.self_attn.o_proj": {
1704
+ "group_size": 128,
1705
+ "bits": 2,
1706
+ "mode": "affine"
1707
+ },
1708
+ "model.layers.5.self_attn.q_proj": {
1709
+ "group_size": 128,
1710
+ "bits": 2,
1711
+ "mode": "affine"
1712
+ },
1713
+ "model.layers.5.self_attn.v_proj": {
1714
+ "group_size": 128,
1715
+ "bits": 2,
1716
+ "mode": "affine"
1717
+ },
1718
+ "model.layers.6.mlp.down_proj": {
1719
+ "group_size": 64,
1720
+ "bits": 8,
1721
+ "mode": "affine"
1722
+ },
1723
+ "model.layers.6.mlp.gate_proj": {
1724
+ "group_size": 128,
1725
+ "bits": 2,
1726
+ "mode": "affine"
1727
+ },
1728
+ "model.layers.6.mlp.up_proj": {
1729
+ "group_size": 128,
1730
+ "bits": 2,
1731
+ "mode": "affine"
1732
+ },
1733
+ "model.layers.6.self_attn.k_proj": {
1734
+ "group_size": 128,
1735
+ "bits": 2,
1736
+ "mode": "affine"
1737
+ },
1738
+ "model.layers.6.self_attn.o_proj": {
1739
+ "group_size": 128,
1740
+ "bits": 2,
1741
+ "mode": "affine"
1742
+ },
1743
+ "model.layers.6.self_attn.q_proj": {
1744
+ "group_size": 128,
1745
+ "bits": 2,
1746
+ "mode": "affine"
1747
+ },
1748
+ "model.layers.6.self_attn.v_proj": {
1749
+ "group_size": 128,
1750
+ "bits": 2,
1751
+ "mode": "affine"
1752
+ },
1753
+ "model.layers.7.mlp.down_proj": {
1754
+ "group_size": 64,
1755
+ "bits": 8,
1756
+ "mode": "affine"
1757
+ },
1758
+ "model.layers.7.mlp.gate_proj": {
1759
+ "group_size": 128,
1760
+ "bits": 2,
1761
+ "mode": "affine"
1762
+ },
1763
+ "model.layers.7.mlp.up_proj": {
1764
+ "group_size": 128,
1765
+ "bits": 2,
1766
+ "mode": "affine"
1767
+ },
1768
+ "model.layers.7.self_attn.k_proj": {
1769
+ "group_size": 128,
1770
+ "bits": 2,
1771
+ "mode": "affine"
1772
+ },
1773
+ "model.layers.7.self_attn.o_proj": {
1774
+ "group_size": 128,
1775
+ "bits": 2,
1776
+ "mode": "affine"
1777
+ },
1778
+ "model.layers.7.self_attn.q_proj": {
1779
+ "group_size": 128,
1780
+ "bits": 2,
1781
+ "mode": "affine"
1782
+ },
1783
+ "model.layers.7.self_attn.v_proj": {
1784
+ "group_size": 128,
1785
+ "bits": 2,
1786
+ "mode": "affine"
1787
+ },
1788
+ "model.layers.8.mlp.down_proj": {
1789
+ "group_size": 64,
1790
+ "bits": 8,
1791
+ "mode": "affine"
1792
+ },
1793
+ "model.layers.8.mlp.gate_proj": {
1794
+ "group_size": 128,
1795
+ "bits": 2,
1796
+ "mode": "affine"
1797
+ },
1798
+ "model.layers.8.mlp.up_proj": {
1799
+ "group_size": 128,
1800
+ "bits": 2,
1801
+ "mode": "affine"
1802
+ },
1803
+ "model.layers.8.self_attn.k_proj": {
1804
+ "group_size": 128,
1805
+ "bits": 2,
1806
+ "mode": "affine"
1807
+ },
1808
+ "model.layers.8.self_attn.o_proj": {
1809
+ "group_size": 128,
1810
+ "bits": 2,
1811
+ "mode": "affine"
1812
+ },
1813
+ "model.layers.8.self_attn.q_proj": {
1814
+ "group_size": 128,
1815
+ "bits": 2,
1816
+ "mode": "affine"
1817
+ },
1818
+ "model.layers.8.self_attn.v_proj": {
1819
+ "group_size": 128,
1820
+ "bits": 2,
1821
+ "mode": "affine"
1822
+ },
1823
+ "model.layers.9.mlp.down_proj": {
1824
+ "group_size": 64,
1825
+ "bits": 8,
1826
+ "mode": "affine"
1827
+ },
1828
+ "model.layers.9.mlp.gate_proj": {
1829
+ "group_size": 128,
1830
+ "bits": 2,
1831
+ "mode": "affine"
1832
+ },
1833
+ "model.layers.9.mlp.up_proj": {
1834
+ "group_size": 128,
1835
+ "bits": 2,
1836
+ "mode": "affine"
1837
+ },
1838
+ "model.layers.9.self_attn.k_proj": {
1839
+ "group_size": 128,
1840
+ "bits": 2,
1841
+ "mode": "affine"
1842
+ },
1843
+ "model.layers.9.self_attn.o_proj": {
1844
+ "group_size": 128,
1845
+ "bits": 2,
1846
+ "mode": "affine"
1847
+ },
1848
+ "model.layers.9.self_attn.q_proj": {
1849
+ "group_size": 128,
1850
+ "bits": 2,
1851
+ "mode": "affine"
1852
+ },
1853
+ "model.layers.9.self_attn.v_proj": {
1854
+ "group_size": 128,
1855
+ "bits": 2,
1856
+ "mode": "affine"
1857
+ },
1858
+ "model.layers.14.mlp.down_proj": {
1859
+ "group_size": 64,
1860
+ "bits": 8,
1861
+ "mode": "affine"
1862
+ },
1863
+ "model.layers.14.mlp.gate_proj": {
1864
+ "group_size": 128,
1865
+ "bits": 2,
1866
+ "mode": "affine"
1867
+ },
1868
+ "model.layers.14.mlp.up_proj": {
1869
+ "group_size": 128,
1870
+ "bits": 2,
1871
+ "mode": "affine"
1872
+ },
1873
+ "model.layers.14.self_attn.k_proj": {
1874
+ "group_size": 128,
1875
+ "bits": 2,
1876
+ "mode": "affine"
1877
+ },
1878
+ "model.layers.14.self_attn.o_proj": {
1879
+ "group_size": 64,
1880
+ "bits": 6,
1881
+ "mode": "affine"
1882
+ },
1883
+ "model.layers.14.self_attn.q_proj": {
1884
+ "group_size": 128,
1885
+ "bits": 2,
1886
+ "mode": "affine"
1887
+ },
1888
+ "model.layers.14.self_attn.v_proj": {
1889
+ "group_size": 128,
1890
+ "bits": 2,
1891
+ "mode": "affine"
1892
+ },
1893
+ "model.layers.15.mlp.down_proj": {
1894
+ "group_size": 64,
1895
+ "bits": 8,
1896
+ "mode": "affine"
1897
+ },
1898
+ "model.layers.15.mlp.gate_proj": {
1899
+ "group_size": 128,
1900
+ "bits": 2,
1901
+ "mode": "affine"
1902
+ },
1903
+ "model.layers.15.mlp.up_proj": {
1904
+ "group_size": 128,
1905
+ "bits": 2,
1906
+ "mode": "affine"
1907
+ },
1908
+ "model.layers.15.self_attn.k_proj": {
1909
+ "group_size": 128,
1910
+ "bits": 2,
1911
+ "mode": "affine"
1912
+ },
1913
+ "model.layers.15.self_attn.o_proj": {
1914
+ "group_size": 64,
1915
+ "bits": 6,
1916
+ "mode": "affine"
1917
+ },
1918
+ "model.layers.15.self_attn.q_proj": {
1919
+ "group_size": 128,
1920
+ "bits": 2,
1921
+ "mode": "affine"
1922
+ },
1923
+ "model.layers.15.self_attn.v_proj": {
1924
+ "group_size": 128,
1925
+ "bits": 2,
1926
+ "mode": "affine"
1927
+ },
1928
+ "model.layers.16.mlp.down_proj": {
1929
+ "group_size": 64,
1930
+ "bits": 8,
1931
+ "mode": "affine"
1932
+ },
1933
+ "model.layers.16.mlp.gate_proj": {
1934
+ "group_size": 128,
1935
+ "bits": 2,
1936
+ "mode": "affine"
1937
+ },
1938
+ "model.layers.16.mlp.up_proj": {
1939
+ "group_size": 128,
1940
+ "bits": 2,
1941
+ "mode": "affine"
1942
+ },
1943
+ "model.layers.16.self_attn.k_proj": {
1944
+ "group_size": 128,
1945
+ "bits": 2,
1946
+ "mode": "affine"
1947
+ },
1948
+ "model.layers.16.self_attn.o_proj": {
1949
+ "group_size": 64,
1950
+ "bits": 6,
1951
+ "mode": "affine"
1952
+ },
1953
+ "model.layers.16.self_attn.q_proj": {
1954
+ "group_size": 128,
1955
+ "bits": 2,
1956
+ "mode": "affine"
1957
+ },
1958
+ "model.layers.16.self_attn.v_proj": {
1959
+ "group_size": 128,
1960
+ "bits": 2,
1961
+ "mode": "affine"
1962
+ },
1963
+ "model.layers.17.mlp.down_proj": {
1964
+ "group_size": 64,
1965
+ "bits": 8,
1966
+ "mode": "affine"
1967
+ },
1968
+ "model.layers.17.mlp.gate_proj": {
1969
+ "group_size": 128,
1970
+ "bits": 2,
1971
+ "mode": "affine"
1972
+ },
1973
+ "model.layers.17.mlp.up_proj": {
1974
+ "group_size": 128,
1975
+ "bits": 2,
1976
+ "mode": "affine"
1977
+ },
1978
+ "model.layers.17.self_attn.k_proj": {
1979
+ "group_size": 128,
1980
+ "bits": 2,
1981
+ "mode": "affine"
1982
+ },
1983
+ "model.layers.17.self_attn.o_proj": {
1984
+ "group_size": 64,
1985
+ "bits": 6,
1986
+ "mode": "affine"
1987
+ },
1988
+ "model.layers.17.self_attn.q_proj": {
1989
+ "group_size": 128,
1990
+ "bits": 2,
1991
+ "mode": "affine"
1992
+ },
1993
+ "model.layers.17.self_attn.v_proj": {
1994
+ "group_size": 128,
1995
+ "bits": 2,
1996
+ "mode": "affine"
1997
+ },
1998
+ "model.layers.18.mlp.down_proj": {
1999
+ "group_size": 64,
2000
+ "bits": 8,
2001
+ "mode": "affine"
2002
+ },
2003
+ "model.layers.18.mlp.gate_proj": {
2004
+ "group_size": 128,
2005
+ "bits": 2,
2006
+ "mode": "affine"
2007
+ },
2008
+ "model.layers.18.mlp.up_proj": {
2009
+ "group_size": 128,
2010
+ "bits": 2,
2011
+ "mode": "affine"
2012
+ },
2013
+ "model.layers.18.self_attn.k_proj": {
2014
+ "group_size": 128,
2015
+ "bits": 2,
2016
+ "mode": "affine"
2017
+ },
2018
+ "model.layers.18.self_attn.o_proj": {
2019
+ "group_size": 64,
2020
+ "bits": 6,
2021
+ "mode": "affine"
2022
+ },
2023
+ "model.layers.18.self_attn.q_proj": {
2024
+ "group_size": 128,
2025
+ "bits": 2,
2026
+ "mode": "affine"
2027
+ },
2028
+ "model.layers.18.self_attn.v_proj": {
2029
+ "group_size": 128,
2030
+ "bits": 2,
2031
+ "mode": "affine"
2032
+ },
2033
+ "model.layers.19.mlp.down_proj": {
2034
+ "group_size": 64,
2035
+ "bits": 8,
2036
+ "mode": "affine"
2037
+ },
2038
+ "model.layers.19.mlp.gate_proj": {
2039
+ "group_size": 128,
2040
+ "bits": 2,
2041
+ "mode": "affine"
2042
+ },
2043
+ "model.layers.19.mlp.up_proj": {
2044
+ "group_size": 128,
2045
+ "bits": 2,
2046
+ "mode": "affine"
2047
+ },
2048
+ "model.layers.19.self_attn.k_proj": {
2049
+ "group_size": 128,
2050
+ "bits": 2,
2051
+ "mode": "affine"
2052
+ },
2053
+ "model.layers.19.self_attn.o_proj": {
2054
+ "group_size": 64,
2055
+ "bits": 6,
2056
+ "mode": "affine"
2057
+ },
2058
+ "model.layers.19.self_attn.q_proj": {
2059
+ "group_size": 128,
2060
+ "bits": 2,
2061
+ "mode": "affine"
2062
+ },
2063
+ "model.layers.19.self_attn.v_proj": {
2064
+ "group_size": 128,
2065
+ "bits": 2,
2066
+ "mode": "affine"
2067
+ },
2068
+ "model.layers.20.mlp.down_proj": {
2069
+ "group_size": 64,
2070
+ "bits": 8,
2071
+ "mode": "affine"
2072
+ },
2073
+ "model.layers.20.mlp.gate_proj": {
2074
+ "group_size": 128,
2075
+ "bits": 2,
2076
+ "mode": "affine"
2077
+ },
2078
+ "model.layers.20.mlp.up_proj": {
2079
+ "group_size": 128,
2080
+ "bits": 2,
2081
+ "mode": "affine"
2082
+ },
2083
+ "model.layers.20.self_attn.k_proj": {
2084
+ "group_size": 128,
2085
+ "bits": 2,
2086
+ "mode": "affine"
2087
+ },
2088
+ "model.layers.20.self_attn.o_proj": {
2089
+ "group_size": 64,
2090
+ "bits": 6,
2091
+ "mode": "affine"
2092
+ },
2093
+ "model.layers.20.self_attn.q_proj": {
2094
+ "group_size": 128,
2095
+ "bits": 2,
2096
+ "mode": "affine"
2097
+ },
2098
+ "model.layers.20.self_attn.v_proj": {
2099
+ "group_size": 128,
2100
+ "bits": 2,
2101
+ "mode": "affine"
2102
+ },
2103
+ "model.layers.21.mlp.down_proj": {
2104
+ "group_size": 64,
2105
+ "bits": 8,
2106
+ "mode": "affine"
2107
+ },
2108
+ "model.layers.21.mlp.gate_proj": {
2109
+ "group_size": 128,
2110
+ "bits": 2,
2111
+ "mode": "affine"
2112
+ },
2113
+ "model.layers.21.mlp.up_proj": {
2114
+ "group_size": 128,
2115
+ "bits": 2,
2116
+ "mode": "affine"
2117
+ },
2118
+ "model.layers.21.self_attn.k_proj": {
2119
+ "group_size": 128,
2120
+ "bits": 2,
2121
+ "mode": "affine"
2122
+ },
2123
+ "model.layers.21.self_attn.o_proj": {
2124
+ "group_size": 64,
2125
+ "bits": 6,
2126
+ "mode": "affine"
2127
+ },
2128
+ "model.layers.21.self_attn.q_proj": {
2129
+ "group_size": 128,
2130
+ "bits": 2,
2131
+ "mode": "affine"
2132
+ },
2133
+ "model.layers.21.self_attn.v_proj": {
2134
+ "group_size": 128,
2135
+ "bits": 2,
2136
+ "mode": "affine"
2137
+ },
2138
+ "model.layers.22.mlp.down_proj": {
2139
+ "group_size": 64,
2140
+ "bits": 8,
2141
+ "mode": "affine"
2142
+ },
2143
+ "model.layers.22.mlp.gate_proj": {
2144
+ "group_size": 128,
2145
+ "bits": 2,
2146
+ "mode": "affine"
2147
+ },
2148
+ "model.layers.22.mlp.up_proj": {
2149
+ "group_size": 128,
2150
+ "bits": 2,
2151
+ "mode": "affine"
2152
+ },
2153
+ "model.layers.22.self_attn.k_proj": {
2154
+ "group_size": 128,
2155
+ "bits": 2,
2156
+ "mode": "affine"
2157
+ },
2158
+ "model.layers.22.self_attn.o_proj": {
2159
+ "group_size": 64,
2160
+ "bits": 6,
2161
+ "mode": "affine"
2162
+ },
2163
+ "model.layers.22.self_attn.q_proj": {
2164
+ "group_size": 128,
2165
+ "bits": 2,
2166
+ "mode": "affine"
2167
+ },
2168
+ "model.layers.22.self_attn.v_proj": {
2169
+ "group_size": 128,
2170
+ "bits": 2,
2171
+ "mode": "affine"
2172
+ },
2173
+ "model.layers.23.mlp.down_proj": {
2174
+ "group_size": 64,
2175
+ "bits": 8,
2176
+ "mode": "affine"
2177
+ },
2178
+ "model.layers.23.mlp.gate_proj": {
2179
+ "group_size": 128,
2180
+ "bits": 2,
2181
+ "mode": "affine"
2182
+ },
2183
+ "model.layers.23.mlp.up_proj": {
2184
+ "group_size": 128,
2185
+ "bits": 2,
2186
+ "mode": "affine"
2187
+ },
2188
+ "model.layers.23.self_attn.k_proj": {
2189
+ "group_size": 128,
2190
+ "bits": 2,
2191
+ "mode": "affine"
2192
+ },
2193
+ "model.layers.23.self_attn.o_proj": {
2194
+ "group_size": 64,
2195
+ "bits": 6,
2196
+ "mode": "affine"
2197
+ },
2198
+ "model.layers.23.self_attn.q_proj": {
2199
+ "group_size": 128,
2200
+ "bits": 2,
2201
+ "mode": "affine"
2202
+ },
2203
+ "model.layers.23.self_attn.v_proj": {
2204
+ "group_size": 128,
2205
+ "bits": 2,
2206
+ "mode": "affine"
2207
+ },
2208
+ "model.layers.24.mlp.down_proj": {
2209
+ "group_size": 64,
2210
+ "bits": 8,
2211
+ "mode": "affine"
2212
+ },
2213
+ "model.layers.24.mlp.gate_proj": {
2214
+ "group_size": 128,
2215
+ "bits": 2,
2216
+ "mode": "affine"
2217
+ },
2218
+ "model.layers.24.mlp.up_proj": {
2219
+ "group_size": 128,
2220
+ "bits": 2,
2221
+ "mode": "affine"
2222
+ },
2223
+ "model.layers.24.self_attn.k_proj": {
2224
+ "group_size": 128,
2225
+ "bits": 2,
2226
+ "mode": "affine"
2227
+ },
2228
+ "model.layers.24.self_attn.o_proj": {
2229
+ "group_size": 64,
2230
+ "bits": 6,
2231
+ "mode": "affine"
2232
+ },
2233
+ "model.layers.24.self_attn.q_proj": {
2234
+ "group_size": 128,
2235
+ "bits": 2,
2236
+ "mode": "affine"
2237
+ },
2238
+ "model.layers.24.self_attn.v_proj": {
2239
+ "group_size": 128,
2240
+ "bits": 2,
2241
+ "mode": "affine"
2242
+ },
2243
+ "model.layers.25.mlp.down_proj": {
2244
+ "group_size": 64,
2245
+ "bits": 8,
2246
+ "mode": "affine"
2247
+ },
2248
+ "model.layers.25.mlp.gate_proj": {
2249
+ "group_size": 128,
2250
+ "bits": 2,
2251
+ "mode": "affine"
2252
+ },
2253
+ "model.layers.25.mlp.up_proj": {
2254
+ "group_size": 128,
2255
+ "bits": 2,
2256
+ "mode": "affine"
2257
+ },
2258
+ "model.layers.25.self_attn.k_proj": {
2259
+ "group_size": 128,
2260
+ "bits": 2,
2261
+ "mode": "affine"
2262
+ },
2263
+ "model.layers.25.self_attn.o_proj": {
2264
+ "group_size": 64,
2265
+ "bits": 6,
2266
+ "mode": "affine"
2267
+ },
2268
+ "model.layers.25.self_attn.q_proj": {
2269
+ "group_size": 128,
2270
+ "bits": 2,
2271
+ "mode": "affine"
2272
+ },
2273
+ "model.layers.25.self_attn.v_proj": {
2274
+ "group_size": 128,
2275
+ "bits": 2,
2276
+ "mode": "affine"
2277
+ },
2278
+ "model.layers.26.mlp.down_proj": {
2279
+ "group_size": 64,
2280
+ "bits": 8,
2281
+ "mode": "affine"
2282
+ },
2283
+ "model.layers.26.mlp.gate_proj": {
2284
+ "group_size": 128,
2285
+ "bits": 2,
2286
+ "mode": "affine"
2287
+ },
2288
+ "model.layers.26.mlp.up_proj": {
2289
+ "group_size": 128,
2290
+ "bits": 2,
2291
+ "mode": "affine"
2292
+ },
2293
+ "model.layers.26.self_attn.k_proj": {
2294
+ "group_size": 128,
2295
+ "bits": 2,
2296
+ "mode": "affine"
2297
+ },
2298
+ "model.layers.26.self_attn.o_proj": {
2299
+ "group_size": 64,
2300
+ "bits": 6,
2301
+ "mode": "affine"
2302
+ },
2303
+ "model.layers.26.self_attn.q_proj": {
2304
+ "group_size": 128,
2305
+ "bits": 2,
2306
+ "mode": "affine"
2307
+ },
2308
+ "model.layers.26.self_attn.v_proj": {
2309
+ "group_size": 128,
2310
+ "bits": 2,
2311
+ "mode": "affine"
2312
+ },
2313
+ "model.layers.27.mlp.down_proj": {
2314
+ "group_size": 64,
2315
+ "bits": 8,
2316
+ "mode": "affine"
2317
+ },
2318
+ "model.layers.27.mlp.gate_proj": {
2319
+ "group_size": 128,
2320
+ "bits": 2,
2321
+ "mode": "affine"
2322
+ },
2323
+ "model.layers.27.mlp.up_proj": {
2324
+ "group_size": 128,
2325
+ "bits": 2,
2326
+ "mode": "affine"
2327
+ },
2328
+ "model.layers.27.self_attn.k_proj": {
2329
+ "group_size": 128,
2330
+ "bits": 2,
2331
+ "mode": "affine"
2332
+ },
2333
+ "model.layers.27.self_attn.o_proj": {
2334
+ "group_size": 64,
2335
+ "bits": 6,
2336
+ "mode": "affine"
2337
+ },
2338
+ "model.layers.27.self_attn.q_proj": {
2339
+ "group_size": 128,
2340
+ "bits": 2,
2341
+ "mode": "affine"
2342
+ },
2343
+ "model.layers.27.self_attn.v_proj": {
2344
+ "group_size": 128,
2345
+ "bits": 2,
2346
+ "mode": "affine"
2347
+ },
2348
+ "model.layers.28.mlp.down_proj": {
2349
+ "group_size": 64,
2350
+ "bits": 8,
2351
+ "mode": "affine"
2352
+ },
2353
+ "model.layers.28.mlp.gate_proj": {
2354
+ "group_size": 128,
2355
+ "bits": 2,
2356
+ "mode": "affine"
2357
+ },
2358
+ "model.layers.28.mlp.up_proj": {
2359
+ "group_size": 128,
2360
+ "bits": 2,
2361
+ "mode": "affine"
2362
+ },
2363
+ "model.layers.28.self_attn.k_proj": {
2364
+ "group_size": 128,
2365
+ "bits": 2,
2366
+ "mode": "affine"
2367
+ },
2368
+ "model.layers.28.self_attn.o_proj": {
2369
+ "group_size": 64,
2370
+ "bits": 6,
2371
+ "mode": "affine"
2372
+ },
2373
+ "model.layers.28.self_attn.q_proj": {
2374
+ "group_size": 128,
2375
+ "bits": 2,
2376
+ "mode": "affine"
2377
+ },
2378
+ "model.layers.28.self_attn.v_proj": {
2379
+ "group_size": 128,
2380
+ "bits": 2,
2381
+ "mode": "affine"
2382
+ },
2383
+ "model.layers.29.mlp.down_proj": {
2384
+ "group_size": 64,
2385
+ "bits": 8,
2386
+ "mode": "affine"
2387
+ },
2388
+ "model.layers.29.mlp.gate_proj": {
2389
+ "group_size": 128,
2390
+ "bits": 2,
2391
+ "mode": "affine"
2392
+ },
2393
+ "model.layers.29.mlp.up_proj": {
2394
+ "group_size": 128,
2395
+ "bits": 2,
2396
+ "mode": "affine"
2397
+ },
2398
+ "model.layers.29.self_attn.k_proj": {
2399
+ "group_size": 128,
2400
+ "bits": 2,
2401
+ "mode": "affine"
2402
+ },
2403
+ "model.layers.29.self_attn.o_proj": {
2404
+ "group_size": 64,
2405
+ "bits": 6,
2406
+ "mode": "affine"
2407
+ },
2408
+ "model.layers.29.self_attn.q_proj": {
2409
+ "group_size": 128,
2410
+ "bits": 2,
2411
+ "mode": "affine"
2412
+ },
2413
+ "model.layers.29.self_attn.v_proj": {
2414
+ "group_size": 128,
2415
+ "bits": 2,
2416
+ "mode": "affine"
2417
+ },
2418
+ "model.layers.30.mlp.down_proj": {
2419
+ "group_size": 64,
2420
+ "bits": 8,
2421
+ "mode": "affine"
2422
+ },
2423
+ "model.layers.30.mlp.gate_proj": {
2424
+ "group_size": 128,
2425
+ "bits": 2,
2426
+ "mode": "affine"
2427
+ },
2428
+ "model.layers.30.mlp.up_proj": {
2429
+ "group_size": 128,
2430
+ "bits": 2,
2431
+ "mode": "affine"
2432
+ },
2433
+ "model.layers.30.self_attn.k_proj": {
2434
+ "group_size": 128,
2435
+ "bits": 2,
2436
+ "mode": "affine"
2437
+ },
2438
+ "model.layers.30.self_attn.o_proj": {
2439
+ "group_size": 64,
2440
+ "bits": 6,
2441
+ "mode": "affine"
2442
+ },
2443
+ "model.layers.30.self_attn.q_proj": {
2444
+ "group_size": 128,
2445
+ "bits": 2,
2446
+ "mode": "affine"
2447
+ },
2448
+ "model.layers.30.self_attn.v_proj": {
2449
+ "group_size": 128,
2450
+ "bits": 2,
2451
+ "mode": "affine"
2452
+ },
2453
+ "model.layers.31.mlp.down_proj": {
2454
+ "group_size": 64,
2455
+ "bits": 8,
2456
+ "mode": "affine"
2457
+ },
2458
+ "model.layers.31.mlp.gate_proj": {
2459
+ "group_size": 128,
2460
+ "bits": 2,
2461
+ "mode": "affine"
2462
+ },
2463
+ "model.layers.31.mlp.up_proj": {
2464
+ "group_size": 128,
2465
+ "bits": 2,
2466
+ "mode": "affine"
2467
+ },
2468
+ "model.layers.31.self_attn.k_proj": {
2469
+ "group_size": 128,
2470
+ "bits": 2,
2471
+ "mode": "affine"
2472
+ },
2473
+ "model.layers.31.self_attn.o_proj": {
2474
+ "group_size": 64,
2475
+ "bits": 6,
2476
+ "mode": "affine"
2477
+ },
2478
+ "model.layers.31.self_attn.q_proj": {
2479
+ "group_size": 128,
2480
+ "bits": 2,
2481
+ "mode": "affine"
2482
+ },
2483
+ "model.layers.31.self_attn.v_proj": {
2484
+ "group_size": 128,
2485
+ "bits": 2,
2486
+ "mode": "affine"
2487
+ },
2488
+ "model.layers.32.mlp.down_proj": {
2489
+ "group_size": 64,
2490
+ "bits": 8,
2491
+ "mode": "affine"
2492
+ },
2493
+ "model.layers.32.mlp.gate_proj": {
2494
+ "group_size": 128,
2495
+ "bits": 2,
2496
+ "mode": "affine"
2497
+ },
2498
+ "model.layers.32.mlp.up_proj": {
2499
+ "group_size": 128,
2500
+ "bits": 2,
2501
+ "mode": "affine"
2502
+ },
2503
+ "model.layers.32.self_attn.k_proj": {
2504
+ "group_size": 128,
2505
+ "bits": 2,
2506
+ "mode": "affine"
2507
+ },
2508
+ "model.layers.32.self_attn.o_proj": {
2509
+ "group_size": 64,
2510
+ "bits": 6,
2511
+ "mode": "affine"
2512
+ },
2513
+ "model.layers.32.self_attn.q_proj": {
2514
+ "group_size": 128,
2515
+ "bits": 2,
2516
+ "mode": "affine"
2517
+ },
2518
+ "model.layers.32.self_attn.v_proj": {
2519
+ "group_size": 128,
2520
+ "bits": 2,
2521
+ "mode": "affine"
2522
+ },
2523
+ "model.layers.33.mlp.down_proj": {
2524
+ "group_size": 64,
2525
+ "bits": 8,
2526
+ "mode": "affine"
2527
+ },
2528
+ "model.layers.33.mlp.gate_proj": {
2529
+ "group_size": 128,
2530
+ "bits": 2,
2531
+ "mode": "affine"
2532
+ },
2533
+ "model.layers.33.mlp.up_proj": {
2534
+ "group_size": 128,
2535
+ "bits": 2,
2536
+ "mode": "affine"
2537
+ },
2538
+ "model.layers.33.self_attn.k_proj": {
2539
+ "group_size": 128,
2540
+ "bits": 2,
2541
+ "mode": "affine"
2542
+ },
2543
+ "model.layers.33.self_attn.o_proj": {
2544
+ "group_size": 64,
2545
+ "bits": 6,
2546
+ "mode": "affine"
2547
+ },
2548
+ "model.layers.33.self_attn.q_proj": {
2549
+ "group_size": 128,
2550
+ "bits": 2,
2551
+ "mode": "affine"
2552
+ },
2553
+ "model.layers.33.self_attn.v_proj": {
2554
+ "group_size": 128,
2555
+ "bits": 2,
2556
+ "mode": "affine"
2557
+ },
2558
+ "model.layers.34.mlp.down_proj": {
2559
+ "group_size": 64,
2560
+ "bits": 8,
2561
+ "mode": "affine"
2562
+ },
2563
+ "model.layers.34.mlp.gate_proj": {
2564
+ "group_size": 128,
2565
+ "bits": 2,
2566
+ "mode": "affine"
2567
+ },
2568
+ "model.layers.34.mlp.up_proj": {
2569
+ "group_size": 128,
2570
+ "bits": 2,
2571
+ "mode": "affine"
2572
+ },
2573
+ "model.layers.34.self_attn.k_proj": {
2574
+ "group_size": 128,
2575
+ "bits": 2,
2576
+ "mode": "affine"
2577
+ },
2578
+ "model.layers.34.self_attn.o_proj": {
2579
+ "group_size": 64,
2580
+ "bits": 6,
2581
+ "mode": "affine"
2582
+ },
2583
+ "model.layers.34.self_attn.q_proj": {
2584
+ "group_size": 128,
2585
+ "bits": 2,
2586
+ "mode": "affine"
2587
+ },
2588
+ "model.layers.34.self_attn.v_proj": {
2589
+ "group_size": 128,
2590
+ "bits": 2,
2591
+ "mode": "affine"
2592
+ },
2593
+ "model.layers.35.mlp.down_proj": {
2594
+ "group_size": 64,
2595
+ "bits": 8,
2596
+ "mode": "affine"
2597
+ },
2598
+ "model.layers.35.mlp.gate_proj": {
2599
+ "group_size": 128,
2600
+ "bits": 2,
2601
+ "mode": "affine"
2602
+ },
2603
+ "model.layers.35.mlp.up_proj": {
2604
+ "group_size": 128,
2605
+ "bits": 2,
2606
+ "mode": "affine"
2607
+ },
2608
+ "model.layers.35.self_attn.k_proj": {
2609
+ "group_size": 128,
2610
+ "bits": 2,
2611
+ "mode": "affine"
2612
+ },
2613
+ "model.layers.35.self_attn.o_proj": {
2614
+ "group_size": 64,
2615
+ "bits": 6,
2616
+ "mode": "affine"
2617
+ },
2618
+ "model.layers.35.self_attn.q_proj": {
2619
+ "group_size": 128,
2620
+ "bits": 2,
2621
+ "mode": "affine"
2622
+ },
2623
+ "model.layers.35.self_attn.v_proj": {
2624
+ "group_size": 128,
2625
+ "bits": 2,
2626
+ "mode": "affine"
2627
+ }
2628
  }
2629
  }
generation_config.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_from_model_config": true,
3
+ "eos_token_id": 151645,
4
+ "output_attentions": false,
5
+ "output_hidden_states": false,
6
+ "pad_token_id": 151643,
7
+ "transformers_version": "5.7.0",
8
+ "use_cache": true
9
+ }
model.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:4b6c966d908634f4c8d8ceae390eaa49025bb5949428a0e13dc12e4cf08a1292
3
- size 2303661704
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:11fd04f80e5d4ff16aa6f67221b426b71954ff2f55a657c0b61bc81fcb0975f3
3
+ size 3933149008
model.safetensors.index.json CHANGED
@@ -1,6 +1,6 @@
1
  {
2
  "metadata": {
3
- "total_size": 2303661704
4
  },
5
  "weight_map": {
6
  "lm_head.weight": "model.safetensors",
 
1
  {
2
  "metadata": {
3
+ "total_size": 3933149008
4
  },
5
  "weight_map": {
6
  "lm_head.weight": "model.safetensors",
tokenizer.json CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:79cb3c783570f1b8fe73b9ed530ae50cae9ce4b6344c0b5edefc50478847eaa4
3
- size 11422817
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9db88711959bb528902347c56759f0e1923e531c7807eeeb79ae28f53a6e89da
3
+ size 11422914
tokenizer_config.json CHANGED
@@ -6,9 +6,15 @@
6
  "eos_token": "<|im_end|>",
7
  "errors": "replace",
8
  "is_local": true,
 
 
9
  "model_max_length": 131072,
 
10
  "pad_token": "<|endoftext|>",
 
 
11
  "split_special_tokens": false,
12
  "tokenizer_class": "Qwen2Tokenizer",
 
13
  "unk_token": null
14
  }
 
6
  "eos_token": "<|im_end|>",
7
  "errors": "replace",
8
  "is_local": true,
9
+ "local_files_only": false,
10
+ "max_length": null,
11
  "model_max_length": 131072,
12
+ "pad_to_multiple_of": null,
13
  "pad_token": "<|endoftext|>",
14
+ "pad_token_type_id": 0,
15
+ "padding_side": "left",
16
  "split_special_tokens": false,
17
  "tokenizer_class": "Qwen2Tokenizer",
18
+ "tool_parser_type": "json_tools",
19
  "unk_token": null
20
  }