diff --git a/.gitattributes b/.gitattributes
index a6344aac8c09253b3b630fb776ae94478aa0275b..52373fe24473b1aa44333d318f578ae6bf04b49b 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
*.zip filter=lfs diff=lfs merge=lfs -text
*.zst filter=lfs diff=lfs merge=lfs -text
*tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text
diff --git a/chat_template.jinja b/chat_template.jinja
new file mode 100644
index 0000000000000000000000000000000000000000..d2d66634c4ab89cbed17faa793b27ac023c209a6
--- /dev/null
+++ b/chat_template.jinja
@@ -0,0 +1,190 @@
+{%- set tool_choice = tool_choice | default('auto') -%}
+{%- set enable_thinking = enable_thinking | default(true) -%}
+{%- set save_history_reasoning_content = save_history_reasoning_content | default(true) -%}
+{%- set ns = namespace(final_messages = [], tool_types = [], last_query_index = -1) -%}
+{%- for message in messages -%}
+ {%- set role = message.get('role') -%}
+ {%- set c = message.get('content') -%}
+ {%- if c is string -%}
+ {%- set ns.final_messages = ns.final_messages + [message] -%}
+ {%- elif c is iterable and c is not mapping -%}
+ {%- set parts = namespace(text=[], thinking=[], tool_calls=[], tool_results=[]) -%}
+ {%- for item in c -%}
+ {%- if item['type'] == 'text' -%}
+ {%- set parts.text = parts.text + [item['text']] -%}
+ {%- elif item['type'] == 'thinking' -%}
+ {%- if item['thinking'] -%}
+ {%- set parts.thinking = parts.thinking + [item['thinking']] -%}
+ {%- endif -%}
+ {%- elif item['type'] == 'tool_use' -%}
+ {%- set parts.tool_calls = parts.tool_calls + [{
+ 'id': item['id'],
+ 'type': 'function',
+ 'function': {
+ 'name': item['name'],
+ 'arguments': item['input']
+ }
+ }] -%}
+ {%- elif item['type'] == 'tool_result' -%}
+ {%- set parts.tool_results = parts.tool_results + [{
+ 'role': 'tool',
+ 'name': item.get('name', ''),
+ 'tool_call_id': item['tool_use_id'],
+ 'content': item['content']
+ }] -%}
+ {%- endif -%}
+ {%- endfor -%}
+ {%- set ns.final_messages = ns.final_messages + parts.tool_results -%}
+ {%- if not (parts.tool_results | length > 0 and parts.text | length == 0 and role == 'user') -%}
+ {%- set ns.final_messages = ns.final_messages + [{
+ 'role': role,
+ 'content': parts.text | join(''),
+ 'reasoning_content': parts.thinking | join(''),
+ 'tool_calls': parts.tool_calls
+ }] -%}
+ {%- endif -%}
+ {%- else -%}
+ {%- set ns.final_messages = ns.final_messages + [message] -%}
+ {%- endif -%}
+{%- endfor -%}
+{%- set messages = ns.final_messages -%}
+
+{%- for idx in range(messages|length) -%}
+ {%- set msg = messages[idx] -%}
+ {%- if msg.role == 'user' -%}
+ {%- set ns.last_query_index = idx -%}
+ {%- endif -%}
+{%- endfor -%}
+
+{%- if tools and tool_choice != 'none' -%}
+ {{- "\n" -}}
+ {{- "# Tools\n" -}}
+ {{- "You have access to the following tools:\n\n" -}}
+ {%- for tool in tools -%}
+ {%- if tool.type not in ns.tool_types -%}
+ {%- set ns.tool_types = ns.tool_types + [tool.type] -%}
+ {{- "## Tool namespace: " ~ tool.type ~ "\n\n" -}}
+ {%- endif -%}
+ {%- if tool.type == 'code_interpreter' -%}
+ {%- set tool = {"type":"code_interpreter","function":{"name":"code_interpreter_preview","description":"The code will be executed in a stateful Jupyter notebook sandbox environment, only supports local computation, data processing, and file operations.\nCode sandbox environment (network isolated) Any external network requests or online API calls are prohibited.\nIf online functionality is needed, please use other permitted tools.\nCode will respond with the output of the execution or time out after 60.0 seconds. ","parameters":{"type":"object","properties":{"language":{"type":"string","description":"The programming language of the code to be executed. Available values: python (Default), java, go, js, ts, c, c++."},"code":{"type":"string","description":"Python code to be executed must not include the following:\n- Importing network libraries such as requests, httplib, etc.\n- Any form of HTTP requests.\n- External API calls.\n- Network port operations. Example: ```python\nimport pandas as pd\npd.DataFrame({'A':[1,2]})\n```"},"timeout":{"type":"number","description":"The maximum execution time of the code, in seconds. Default is 60.0."}}},"required":["code"]}} -%}
+ {%- endif -%}
+ {{- "### Tool name: " + tool.function.name + "\n" -}}
+ {{- "Description: " + tool.function.description + "\n\n" -}}
+ {{- "InputSchema: " + tool.function.parameters | tojson(ensure_ascii=False) + "\n\n" -}}
+ {%- endfor -%}
+ {{- '**Note**: For each function call, output the function name and arguments within the following XML format:\n{function-name}\n{arg-key-1}\n{arg-value-1}\n{arg-key-2}\n{arg-value-2}\n...\n\n' -}}
+ {{- ""-}}
+{%- endif -%}
+
+{%- for msg in messages -%}
+ {%- if msg.role == "system" -%}
+ {{- "" + msg.content -}}
+ {%- elif msg.role == "user" -%}
+ {{- "" -}}
+ {%- if msg["files"] -%}
+ {{- '\n' ~ msg.files | tojson(indent=2) ~ '\n' -}}
+ {%- endif -%}
+ {{- msg.content -}}
+ {%- if save_history_reasoning_content and enable_thinking is not none -%}
+ {%- if loop.last or messages[loop.index0 + 1].role != 'assistant' -%}
+ {# pass #}
+ {%- else -%}
+ {%- if enable_thinking == true -%}
+ {{- " /think_on" -}}
+ {%- if thinking_budget -%}
+ {%- if thinking_budget < 1024 -%}
+ {%- set thinking_budget = 1024 -%}
+ {%- endif -%}
+ {{- "\nthinking_budget: < " ~ thinking_budget ~ "." -}}
+ {%- endif -%}
+ {{- " " -}}
+ {%- elif enable_thinking == false -%}
+ {{- " /think_off " -}}
+ {%- endif -%}
+ {%- endif -%}
+ {%- elif loop.index0 >= ns.last_query_index and enable_thinking is not none -%}
+ {%- if loop.last or messages[loop.index0 + 1].role != 'assistant' -%}
+ {# pass #}
+ {%- else -%}
+ {%- if enable_thinking == true -%}
+ {{- " /think_on" -}}
+ {%- if thinking_budget -%}
+ {%- if thinking_budget < 1024 -%}
+ {%- set thinking_budget = 1024 -%}
+ {%- endif -%}
+ {{- "\nthinking_budget: < " ~ thinking_budget ~ "." -}}
+ {%- endif -%}
+ {{- " " -}}
+ {%- elif enable_thinking == false -%}
+ {{- " /think_off " -}}
+ {%- endif -%}
+ {%- endif -%}
+ {%- endif -%}
+ {%- elif msg.role == "assistant" -%}
+ {{- "" -}}
+ {%- if save_history_reasoning_content == true or loop.index0 > ns.last_query_index -%}
+ {%- if enable_thinking == true -%}
+ {%- if msg.reasoning_content -%}
+ {%- set reasoning_content = msg.reasoning_content | trim -%}
+ {{- "\n" ~ reasoning_content ~ "\n\n" -}}
+ {%- else -%}
+ {{- "\n\n\n" -}}
+ {%- endif -%}
+ {%- elif enable_thinking == false -%}
+ {{- "\n\n\n" -}}
+ {%- endif -%}
+ {%- endif -%}
+ {%- if msg.content -%}
+ {{- msg.content -}}
+ {%- endif -%}
+ {%- if msg.tool_calls -%}
+ {%- for tool_call in msg.tool_calls -%}
+ {{- "" ~ tool_call.function.name ~ "\n" -}}
+ {%- set _args = tool_call.function.arguments -%}
+ {%- for k, v in _args.items() -%}
+ {{- "" ~ k ~ "\n" -}}
+ {{- "" ~ (v if v is string else v | tojson(ensure_ascii=False)) ~ "\n" -}}
+ {% endfor -%}
+ {{- "\n" -}}
+ {%- endfor -%}
+ {%- endif -%}
+ {{- "" -}}
+ {%- elif msg.role == "tool" -%}
+ {%- if messages[loop.index0 - 1].role != "tool" -%}
+ {{- "" -}}
+ {%- endif -%}
+ {{- "" ~ msg.name ~ "\n" ~ msg.content ~ "" -}}
+ {%- if loop.index0 == messages|length - 1 -%}
+ {# pass #}
+ {%- elif (save_history_reasoning_content or loop.index0 > ns.last_query_index) and messages[loop.index0 + 1].role != "tool" -%}
+ {%- if enable_thinking == true -%}
+ {{- " /think_on" -}}
+ {%- if thinking_budget -%}
+ {%- if thinking_budget < 1024 -%}
+ {%- set thinking_budget = 1024 -%}
+ {%- endif -%}
+ {{- "\nthinking_budget: < " ~ thinking_budget ~ "." -}}
+ {%- endif -%}
+ {{- " " -}}
+ {%- elif enable_thinking == false -%}
+ {{- " /think_off " -}}
+ {%- endif -%}
+ {%- endif -%}
+ {%- endif -%}
+{%- endfor -%}
+{%- if add_generation_prompt -%}
+ {%- if enable_thinking == true -%}
+ {{- " /think_on" -}}
+ {%- if thinking_budget -%}
+ {%- if thinking_budget < 1024 -%}
+ {%- set thinking_budget = 1024 -%}
+ {%- endif -%}
+ {{- "\nthinking_budget: < " ~ thinking_budget ~ "." -}}
+ {%- endif -%}
+ {{- " " -}}
+ {%- elif enable_thinking == false -%}
+ {{- " /think_off \n\n\n" -}}
+ {%- else -%}
+ {{- " /think_off \n\n\n" -}}
+ {%- endif -%}
+{%- endif -%}
\ No newline at end of file
diff --git a/config.json b/config.json
new file mode 100644
index 0000000000000000000000000000000000000000..4e0f3b88f2365cd17c2948bb72780b4691da6078
--- /dev/null
+++ b/config.json
@@ -0,0 +1,7942 @@
+{
+ "architectures": [
+ "LongcatCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "attention_method": "MLA",
+ "bos_token_id": 1,
+ "cli_factor": 2,
+ "disable_quant_module": [],
+ "dsa_mtp_cli": true,
+ "eos_token_id": 2,
+ "expert_ffn_hidden_size": 2048,
+ "ffn_hidden_size": 12288,
+ "hidden_size": 8192,
+ "index_head_dim": 128,
+ "index_init_tokens": 16,
+ "index_k_norm_type": "rms",
+ "index_local_tokens": 1024,
+ "index_n_heads": 32,
+ "index_topk": 2048,
+ "kv_lora_rank": 512,
+ "max_position_embeddings": 262144,
+ "mla_scale_kv_lora": true,
+ "mla_scale_q_lora": true,
+ "model_type": "longcat2",
+ "moe_impl": "mix",
+ "moe_switch_token_num": 1024,
+ "moe_topk": 12,
+ "mtp_disable_over_tokenizer": true,
+ "mtp_num_layers": 3,
+ "mtp_replicate_modules": true,
+ "n_routed_experts": 768,
+ "num_attention_heads": 64,
+ "num_layers": 38,
+ "oe_neighbor_num": 5,
+ "oe_split_num": 4,
+ "oe_vocab_size_ratio": 100.567,
+ "q_lora_rank": 1536,
+ "qk_nope_head_dim": 128,
+ "qk_rope_head_dim": 64,
+ "quantization": {
+ "group_size": 64,
+ "bits": 3,
+ "mode": "affine",
+ "model.ngram_embeddings.word_embeddings": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.ngram_embeddings.embedders.0": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.ngram_embeddings.embedders.1": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.ngram_embeddings.embedders.2": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.ngram_embeddings.embedders.3": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.ngram_embeddings.embedders.4": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.ngram_embeddings.embedders.5": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.ngram_embeddings.embedders.6": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.ngram_embeddings.embedders.7": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.ngram_embeddings.embedders.8": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.ngram_embeddings.embedders.9": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.ngram_embeddings.embedders.10": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.ngram_embeddings.embedders.11": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.ngram_embeddings.embedders.12": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.ngram_embeddings.embedders.13": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.ngram_embeddings.embedders.14": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.ngram_embeddings.embedders.15": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.ngram_embeddings.post_projs.0": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.ngram_embeddings.post_projs.1": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.ngram_embeddings.post_projs.2": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.ngram_embeddings.post_projs.3": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.ngram_embeddings.post_projs.4": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.ngram_embeddings.post_projs.5": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.ngram_embeddings.post_projs.6": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.ngram_embeddings.post_projs.7": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.ngram_embeddings.post_projs.8": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.ngram_embeddings.post_projs.9": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.ngram_embeddings.post_projs.10": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.ngram_embeddings.post_projs.11": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.ngram_embeddings.post_projs.12": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.ngram_embeddings.post_projs.13": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.ngram_embeddings.post_projs.14": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.ngram_embeddings.post_projs.15": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.0.mlp.switch_mlp.gate_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.0.mlp.switch_mlp.up_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.0.mlp.switch_mlp.down_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.0.mlp.router.classifier": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.0.self_attn.0.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.0.self_attn.0.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.0.self_attn.0.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.0.self_attn.0.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.0.self_attn.0.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.0.self_attn.0.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.0.self_attn.0.indexer.wq_b": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.0.self_attn.0.indexer.wk": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.0.self_attn.0.indexer.weights_proj": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.0.self_attn.1.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.0.self_attn.1.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.0.self_attn.1.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.0.self_attn.1.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.0.self_attn.1.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.0.self_attn.1.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.0.mlps.0.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.0.mlps.0.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.0.mlps.0.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.0.mlps.1.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.0.mlps.1.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.0.mlps.1.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.1.mlp.switch_mlp.gate_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.1.mlp.switch_mlp.up_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.1.mlp.switch_mlp.down_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.1.mlp.router.classifier": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.1.self_attn.0.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.1.self_attn.0.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.1.self_attn.0.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.1.self_attn.0.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.1.self_attn.0.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.1.self_attn.0.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.1.self_attn.0.indexer.wq_b": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.1.self_attn.0.indexer.wk": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.1.self_attn.0.indexer.weights_proj": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.1.self_attn.1.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.1.self_attn.1.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.1.self_attn.1.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.1.self_attn.1.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.1.self_attn.1.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.1.self_attn.1.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.1.mlps.0.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.1.mlps.0.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.1.mlps.0.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.1.mlps.1.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.1.mlps.1.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.1.mlps.1.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.2.mlp.switch_mlp.gate_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.2.mlp.switch_mlp.up_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.2.mlp.switch_mlp.down_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.2.mlp.router.classifier": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.2.self_attn.0.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.2.self_attn.0.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.2.self_attn.0.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.2.self_attn.0.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.2.self_attn.0.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.2.self_attn.0.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.2.self_attn.0.indexer.wq_b": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.2.self_attn.0.indexer.wk": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.2.self_attn.0.indexer.weights_proj": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.2.self_attn.1.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.2.self_attn.1.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.2.self_attn.1.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.2.self_attn.1.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.2.self_attn.1.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.2.self_attn.1.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.2.mlps.0.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.2.mlps.0.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.2.mlps.0.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.2.mlps.1.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.2.mlps.1.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.2.mlps.1.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.3.mlp.switch_mlp.gate_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.3.mlp.switch_mlp.up_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.3.mlp.switch_mlp.down_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.3.mlp.router.classifier": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.3.self_attn.0.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.3.self_attn.0.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.3.self_attn.0.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.3.self_attn.0.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.3.self_attn.0.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.3.self_attn.0.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.3.self_attn.0.indexer.wq_b": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.3.self_attn.0.indexer.wk": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.3.self_attn.0.indexer.weights_proj": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.3.self_attn.1.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.3.self_attn.1.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.3.self_attn.1.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.3.self_attn.1.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.3.self_attn.1.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.3.self_attn.1.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.3.mlps.0.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.3.mlps.0.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.3.mlps.0.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.3.mlps.1.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.3.mlps.1.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.3.mlps.1.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.4.mlp.switch_mlp.gate_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.4.mlp.switch_mlp.up_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.4.mlp.switch_mlp.down_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.4.mlp.router.classifier": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.4.self_attn.0.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.4.self_attn.0.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.4.self_attn.0.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.4.self_attn.0.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.4.self_attn.0.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.4.self_attn.0.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.4.self_attn.0.indexer.wq_b": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.4.self_attn.0.indexer.wk": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.4.self_attn.0.indexer.weights_proj": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.4.self_attn.1.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.4.self_attn.1.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.4.self_attn.1.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.4.self_attn.1.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.4.self_attn.1.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.4.self_attn.1.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.4.mlps.0.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.4.mlps.0.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.4.mlps.0.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.4.mlps.1.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.4.mlps.1.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.4.mlps.1.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.5.mlp.switch_mlp.gate_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.5.mlp.switch_mlp.up_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.5.mlp.switch_mlp.down_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.5.mlp.router.classifier": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.5.self_attn.0.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.5.self_attn.0.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.5.self_attn.0.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.5.self_attn.0.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.5.self_attn.0.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.5.self_attn.0.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.5.self_attn.0.indexer.wq_b": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.5.self_attn.0.indexer.wk": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.5.self_attn.0.indexer.weights_proj": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.5.self_attn.1.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.5.self_attn.1.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.5.self_attn.1.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.5.self_attn.1.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.5.self_attn.1.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.5.self_attn.1.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.5.mlps.0.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.5.mlps.0.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.5.mlps.0.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.5.mlps.1.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.5.mlps.1.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.5.mlps.1.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.6.mlp.switch_mlp.gate_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.6.mlp.switch_mlp.up_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.6.mlp.switch_mlp.down_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.6.mlp.router.classifier": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.6.self_attn.0.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.6.self_attn.0.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.6.self_attn.0.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.6.self_attn.0.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.6.self_attn.0.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.6.self_attn.0.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.6.self_attn.0.indexer.wq_b": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.6.self_attn.0.indexer.wk": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.6.self_attn.0.indexer.weights_proj": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.6.self_attn.1.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.6.self_attn.1.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.6.self_attn.1.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.6.self_attn.1.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.6.self_attn.1.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.6.self_attn.1.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.6.mlps.0.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.6.mlps.0.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.6.mlps.0.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.6.mlps.1.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.6.mlps.1.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.6.mlps.1.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.7.mlp.switch_mlp.gate_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.7.mlp.switch_mlp.up_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.7.mlp.switch_mlp.down_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.7.mlp.router.classifier": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.7.self_attn.0.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.7.self_attn.0.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.7.self_attn.0.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.7.self_attn.0.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.7.self_attn.0.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.7.self_attn.0.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.7.self_attn.0.indexer.wq_b": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.7.self_attn.0.indexer.wk": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.7.self_attn.0.indexer.weights_proj": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.7.self_attn.1.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.7.self_attn.1.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.7.self_attn.1.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.7.self_attn.1.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.7.self_attn.1.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.7.self_attn.1.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.7.mlps.0.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.7.mlps.0.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.7.mlps.0.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.7.mlps.1.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.7.mlps.1.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.7.mlps.1.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.8.mlp.switch_mlp.gate_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.8.mlp.switch_mlp.up_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.8.mlp.switch_mlp.down_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.8.mlp.router.classifier": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.8.self_attn.0.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.8.self_attn.0.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.8.self_attn.0.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.8.self_attn.0.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.8.self_attn.0.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.8.self_attn.0.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.8.self_attn.0.indexer.wq_b": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.8.self_attn.0.indexer.wk": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.8.self_attn.0.indexer.weights_proj": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.8.self_attn.1.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.8.self_attn.1.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.8.self_attn.1.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.8.self_attn.1.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.8.self_attn.1.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.8.self_attn.1.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.8.mlps.0.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.8.mlps.0.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.8.mlps.0.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.8.mlps.1.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.8.mlps.1.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.8.mlps.1.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.9.mlp.switch_mlp.gate_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.9.mlp.switch_mlp.up_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.9.mlp.switch_mlp.down_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.9.mlp.router.classifier": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.9.self_attn.0.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.9.self_attn.0.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.9.self_attn.0.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.9.self_attn.0.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.9.self_attn.0.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.9.self_attn.0.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.9.self_attn.0.indexer.wq_b": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.9.self_attn.0.indexer.wk": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.9.self_attn.0.indexer.weights_proj": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.9.self_attn.1.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.9.self_attn.1.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.9.self_attn.1.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.9.self_attn.1.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.9.self_attn.1.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.9.self_attn.1.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.9.mlps.0.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.9.mlps.0.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.9.mlps.0.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.9.mlps.1.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.9.mlps.1.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.9.mlps.1.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.10.mlp.switch_mlp.gate_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.10.mlp.switch_mlp.up_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.10.mlp.switch_mlp.down_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.10.mlp.router.classifier": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.10.self_attn.0.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.10.self_attn.0.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.10.self_attn.0.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.10.self_attn.0.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.10.self_attn.0.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.10.self_attn.0.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.10.self_attn.0.indexer.wq_b": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.10.self_attn.0.indexer.wk": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.10.self_attn.0.indexer.weights_proj": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.10.self_attn.1.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.10.self_attn.1.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.10.self_attn.1.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.10.self_attn.1.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.10.self_attn.1.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.10.self_attn.1.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.10.mlps.0.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.10.mlps.0.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.10.mlps.0.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.10.mlps.1.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.10.mlps.1.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.10.mlps.1.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.11.mlp.switch_mlp.gate_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.11.mlp.switch_mlp.up_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.11.mlp.switch_mlp.down_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.11.mlp.router.classifier": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.11.self_attn.0.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.11.self_attn.0.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.11.self_attn.0.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.11.self_attn.0.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.11.self_attn.0.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.11.self_attn.0.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.11.self_attn.0.indexer.wq_b": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.11.self_attn.0.indexer.wk": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.11.self_attn.0.indexer.weights_proj": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.11.self_attn.1.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.11.self_attn.1.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.11.self_attn.1.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.11.self_attn.1.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.11.self_attn.1.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.11.self_attn.1.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.11.mlps.0.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.11.mlps.0.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.11.mlps.0.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.11.mlps.1.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.11.mlps.1.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.11.mlps.1.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.12.mlp.switch_mlp.gate_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.12.mlp.switch_mlp.up_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.12.mlp.switch_mlp.down_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.12.mlp.router.classifier": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.12.self_attn.0.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.12.self_attn.0.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.12.self_attn.0.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.12.self_attn.0.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.12.self_attn.0.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.12.self_attn.0.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.12.self_attn.0.indexer.wq_b": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.12.self_attn.0.indexer.wk": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.12.self_attn.0.indexer.weights_proj": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.12.self_attn.1.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.12.self_attn.1.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.12.self_attn.1.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.12.self_attn.1.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.12.self_attn.1.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.12.self_attn.1.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.12.mlps.0.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.12.mlps.0.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.12.mlps.0.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.12.mlps.1.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.12.mlps.1.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.12.mlps.1.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.13.mlp.switch_mlp.gate_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.13.mlp.switch_mlp.up_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.13.mlp.switch_mlp.down_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.13.mlp.router.classifier": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.13.self_attn.0.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.13.self_attn.0.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.13.self_attn.0.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.13.self_attn.0.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.13.self_attn.0.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.13.self_attn.0.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.13.self_attn.0.indexer.wq_b": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.13.self_attn.0.indexer.wk": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.13.self_attn.0.indexer.weights_proj": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.13.self_attn.1.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.13.self_attn.1.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.13.self_attn.1.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.13.self_attn.1.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.13.self_attn.1.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.13.self_attn.1.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.13.mlps.0.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.13.mlps.0.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.13.mlps.0.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.13.mlps.1.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.13.mlps.1.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.13.mlps.1.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.14.mlp.switch_mlp.gate_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.14.mlp.switch_mlp.up_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.14.mlp.switch_mlp.down_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.14.mlp.router.classifier": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.14.self_attn.0.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.14.self_attn.0.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.14.self_attn.0.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.14.self_attn.0.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.14.self_attn.0.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.14.self_attn.0.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.14.self_attn.0.indexer.wq_b": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.14.self_attn.0.indexer.wk": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.14.self_attn.0.indexer.weights_proj": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.14.self_attn.1.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.14.self_attn.1.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.14.self_attn.1.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.14.self_attn.1.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.14.self_attn.1.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.14.self_attn.1.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.14.mlps.0.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.14.mlps.0.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.14.mlps.0.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.14.mlps.1.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.14.mlps.1.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.14.mlps.1.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.15.mlp.switch_mlp.gate_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.15.mlp.switch_mlp.up_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.15.mlp.switch_mlp.down_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.15.mlp.router.classifier": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.15.self_attn.0.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.15.self_attn.0.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.15.self_attn.0.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.15.self_attn.0.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.15.self_attn.0.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.15.self_attn.0.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.15.self_attn.0.indexer.wq_b": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.15.self_attn.0.indexer.wk": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.15.self_attn.0.indexer.weights_proj": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.15.self_attn.1.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.15.self_attn.1.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.15.self_attn.1.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.15.self_attn.1.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.15.self_attn.1.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.15.self_attn.1.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.15.mlps.0.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.15.mlps.0.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.15.mlps.0.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.15.mlps.1.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.15.mlps.1.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.15.mlps.1.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.16.mlp.switch_mlp.gate_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.16.mlp.switch_mlp.up_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.16.mlp.switch_mlp.down_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.16.mlp.router.classifier": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.16.self_attn.0.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.16.self_attn.0.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.16.self_attn.0.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.16.self_attn.0.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.16.self_attn.0.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.16.self_attn.0.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.16.self_attn.0.indexer.wq_b": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.16.self_attn.0.indexer.wk": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.16.self_attn.0.indexer.weights_proj": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.16.self_attn.1.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.16.self_attn.1.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.16.self_attn.1.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.16.self_attn.1.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.16.self_attn.1.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.16.self_attn.1.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.16.mlps.0.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.16.mlps.0.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.16.mlps.0.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.16.mlps.1.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.16.mlps.1.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.16.mlps.1.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.17.mlp.switch_mlp.gate_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.17.mlp.switch_mlp.up_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.17.mlp.switch_mlp.down_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.17.mlp.router.classifier": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.17.self_attn.0.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.17.self_attn.0.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.17.self_attn.0.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.17.self_attn.0.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.17.self_attn.0.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.17.self_attn.0.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.17.self_attn.0.indexer.wq_b": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.17.self_attn.0.indexer.wk": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.17.self_attn.0.indexer.weights_proj": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.17.self_attn.1.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.17.self_attn.1.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.17.self_attn.1.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.17.self_attn.1.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.17.self_attn.1.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.17.self_attn.1.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.17.mlps.0.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.17.mlps.0.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.17.mlps.0.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.17.mlps.1.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.17.mlps.1.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.17.mlps.1.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.18.mlp.switch_mlp.gate_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.18.mlp.switch_mlp.up_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.18.mlp.switch_mlp.down_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.18.mlp.router.classifier": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.18.self_attn.0.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.18.self_attn.0.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.18.self_attn.0.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.18.self_attn.0.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.18.self_attn.0.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.18.self_attn.0.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.18.self_attn.0.indexer.wq_b": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.18.self_attn.0.indexer.wk": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.18.self_attn.0.indexer.weights_proj": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.18.self_attn.1.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.18.self_attn.1.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.18.self_attn.1.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.18.self_attn.1.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.18.self_attn.1.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.18.self_attn.1.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.18.mlps.0.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.18.mlps.0.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.18.mlps.0.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.18.mlps.1.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.18.mlps.1.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.18.mlps.1.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.19.mlp.switch_mlp.gate_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.19.mlp.switch_mlp.up_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.19.mlp.switch_mlp.down_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.19.mlp.router.classifier": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.19.self_attn.0.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.19.self_attn.0.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.19.self_attn.0.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.19.self_attn.0.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.19.self_attn.0.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.19.self_attn.0.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.19.self_attn.0.indexer.wq_b": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.19.self_attn.0.indexer.wk": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.19.self_attn.0.indexer.weights_proj": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.19.self_attn.1.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.19.self_attn.1.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.19.self_attn.1.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.19.self_attn.1.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.19.self_attn.1.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.19.self_attn.1.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.19.mlps.0.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.19.mlps.0.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.19.mlps.0.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.19.mlps.1.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.19.mlps.1.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.19.mlps.1.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.20.mlp.switch_mlp.gate_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.20.mlp.switch_mlp.up_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.20.mlp.switch_mlp.down_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.20.mlp.router.classifier": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.20.self_attn.0.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.20.self_attn.0.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.20.self_attn.0.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.20.self_attn.0.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.20.self_attn.0.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.20.self_attn.0.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.20.self_attn.0.indexer.wq_b": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.20.self_attn.0.indexer.wk": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.20.self_attn.0.indexer.weights_proj": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.20.self_attn.1.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.20.self_attn.1.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.20.self_attn.1.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.20.self_attn.1.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.20.self_attn.1.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.20.self_attn.1.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.20.mlps.0.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.20.mlps.0.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.20.mlps.0.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.20.mlps.1.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.20.mlps.1.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.20.mlps.1.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.21.mlp.switch_mlp.gate_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.21.mlp.switch_mlp.up_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.21.mlp.switch_mlp.down_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.21.mlp.router.classifier": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.21.self_attn.0.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.21.self_attn.0.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.21.self_attn.0.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.21.self_attn.0.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.21.self_attn.0.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.21.self_attn.0.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.21.self_attn.0.indexer.wq_b": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.21.self_attn.0.indexer.wk": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.21.self_attn.0.indexer.weights_proj": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.21.self_attn.1.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.21.self_attn.1.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.21.self_attn.1.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.21.self_attn.1.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.21.self_attn.1.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.21.self_attn.1.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.21.mlps.0.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.21.mlps.0.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.21.mlps.0.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.21.mlps.1.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.21.mlps.1.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.21.mlps.1.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.22.mlp.switch_mlp.gate_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.22.mlp.switch_mlp.up_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.22.mlp.switch_mlp.down_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.22.mlp.router.classifier": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.22.self_attn.0.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.22.self_attn.0.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.22.self_attn.0.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.22.self_attn.0.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.22.self_attn.0.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.22.self_attn.0.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.22.self_attn.0.indexer.wq_b": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.22.self_attn.0.indexer.wk": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.22.self_attn.0.indexer.weights_proj": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.22.self_attn.1.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.22.self_attn.1.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.22.self_attn.1.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.22.self_attn.1.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.22.self_attn.1.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.22.self_attn.1.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.22.mlps.0.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.22.mlps.0.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.22.mlps.0.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.22.mlps.1.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.22.mlps.1.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.22.mlps.1.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.23.mlp.switch_mlp.gate_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.23.mlp.switch_mlp.up_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.23.mlp.switch_mlp.down_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.23.mlp.router.classifier": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.23.self_attn.0.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.23.self_attn.0.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.23.self_attn.0.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.23.self_attn.0.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.23.self_attn.0.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.23.self_attn.0.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.23.self_attn.0.indexer.wq_b": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.23.self_attn.0.indexer.wk": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.23.self_attn.0.indexer.weights_proj": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.23.self_attn.1.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.23.self_attn.1.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.23.self_attn.1.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.23.self_attn.1.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.23.self_attn.1.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.23.self_attn.1.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.23.mlps.0.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.23.mlps.0.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.23.mlps.0.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.23.mlps.1.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.23.mlps.1.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.23.mlps.1.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.24.mlp.switch_mlp.gate_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.24.mlp.switch_mlp.up_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.24.mlp.switch_mlp.down_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.24.mlp.router.classifier": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.24.self_attn.0.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.24.self_attn.0.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.24.self_attn.0.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.24.self_attn.0.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.24.self_attn.0.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.24.self_attn.0.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.24.self_attn.0.indexer.wq_b": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.24.self_attn.0.indexer.wk": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.24.self_attn.0.indexer.weights_proj": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.24.self_attn.1.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.24.self_attn.1.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.24.self_attn.1.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.24.self_attn.1.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.24.self_attn.1.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.24.self_attn.1.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.24.mlps.0.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.24.mlps.0.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.24.mlps.0.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.24.mlps.1.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.24.mlps.1.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.24.mlps.1.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.25.mlp.switch_mlp.gate_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.25.mlp.switch_mlp.up_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.25.mlp.switch_mlp.down_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.25.mlp.router.classifier": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.25.self_attn.0.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.25.self_attn.0.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.25.self_attn.0.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.25.self_attn.0.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.25.self_attn.0.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.25.self_attn.0.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.25.self_attn.0.indexer.wq_b": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.25.self_attn.0.indexer.wk": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.25.self_attn.0.indexer.weights_proj": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.25.self_attn.1.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.25.self_attn.1.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.25.self_attn.1.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.25.self_attn.1.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.25.self_attn.1.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.25.self_attn.1.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.25.mlps.0.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.25.mlps.0.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.25.mlps.0.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.25.mlps.1.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.25.mlps.1.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.25.mlps.1.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.26.mlp.switch_mlp.gate_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.26.mlp.switch_mlp.up_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.26.mlp.switch_mlp.down_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.26.mlp.router.classifier": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.26.self_attn.0.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.26.self_attn.0.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.26.self_attn.0.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.26.self_attn.0.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.26.self_attn.0.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.26.self_attn.0.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.26.self_attn.0.indexer.wq_b": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.26.self_attn.0.indexer.wk": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.26.self_attn.0.indexer.weights_proj": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.26.self_attn.1.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.26.self_attn.1.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.26.self_attn.1.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.26.self_attn.1.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.26.self_attn.1.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.26.self_attn.1.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.26.mlps.0.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.26.mlps.0.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.26.mlps.0.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.26.mlps.1.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.26.mlps.1.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.26.mlps.1.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.27.mlp.switch_mlp.gate_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.27.mlp.switch_mlp.up_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.27.mlp.switch_mlp.down_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.27.mlp.router.classifier": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.27.self_attn.0.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.27.self_attn.0.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.27.self_attn.0.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.27.self_attn.0.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.27.self_attn.0.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.27.self_attn.0.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.27.self_attn.0.indexer.wq_b": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.27.self_attn.0.indexer.wk": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.27.self_attn.0.indexer.weights_proj": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.27.self_attn.1.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.27.self_attn.1.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.27.self_attn.1.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.27.self_attn.1.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.27.self_attn.1.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.27.self_attn.1.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.27.mlps.0.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.27.mlps.0.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.27.mlps.0.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.27.mlps.1.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.27.mlps.1.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.27.mlps.1.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.28.mlp.switch_mlp.gate_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.28.mlp.switch_mlp.up_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.28.mlp.switch_mlp.down_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.28.mlp.router.classifier": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.28.self_attn.0.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.28.self_attn.0.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.28.self_attn.0.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.28.self_attn.0.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.28.self_attn.0.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.28.self_attn.0.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.28.self_attn.0.indexer.wq_b": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.28.self_attn.0.indexer.wk": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.28.self_attn.0.indexer.weights_proj": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.28.self_attn.1.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.28.self_attn.1.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.28.self_attn.1.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.28.self_attn.1.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.28.self_attn.1.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.28.self_attn.1.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.28.mlps.0.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.28.mlps.0.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.28.mlps.0.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.28.mlps.1.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.28.mlps.1.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.28.mlps.1.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.29.mlp.switch_mlp.gate_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.29.mlp.switch_mlp.up_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.29.mlp.switch_mlp.down_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.29.mlp.router.classifier": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.29.self_attn.0.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.29.self_attn.0.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.29.self_attn.0.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.29.self_attn.0.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.29.self_attn.0.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.29.self_attn.0.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.29.self_attn.0.indexer.wq_b": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.29.self_attn.0.indexer.wk": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.29.self_attn.0.indexer.weights_proj": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.29.self_attn.1.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.29.self_attn.1.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.29.self_attn.1.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.29.self_attn.1.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.29.self_attn.1.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.29.self_attn.1.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.29.mlps.0.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.29.mlps.0.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.29.mlps.0.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.29.mlps.1.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.29.mlps.1.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.29.mlps.1.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.30.mlp.switch_mlp.gate_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.30.mlp.switch_mlp.up_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.30.mlp.switch_mlp.down_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.30.mlp.router.classifier": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.30.self_attn.0.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.30.self_attn.0.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.30.self_attn.0.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.30.self_attn.0.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.30.self_attn.0.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.30.self_attn.0.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.30.self_attn.0.indexer.wq_b": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.30.self_attn.0.indexer.wk": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.30.self_attn.0.indexer.weights_proj": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.30.self_attn.1.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.30.self_attn.1.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.30.self_attn.1.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.30.self_attn.1.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.30.self_attn.1.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.30.self_attn.1.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.30.mlps.0.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.30.mlps.0.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.30.mlps.0.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.30.mlps.1.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.30.mlps.1.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.30.mlps.1.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.31.mlp.switch_mlp.gate_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.31.mlp.switch_mlp.up_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.31.mlp.switch_mlp.down_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.31.mlp.router.classifier": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.31.self_attn.0.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.31.self_attn.0.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.31.self_attn.0.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.31.self_attn.0.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.31.self_attn.0.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.31.self_attn.0.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.31.self_attn.0.indexer.wq_b": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.31.self_attn.0.indexer.wk": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.31.self_attn.0.indexer.weights_proj": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.31.self_attn.1.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.31.self_attn.1.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.31.self_attn.1.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.31.self_attn.1.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.31.self_attn.1.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.31.self_attn.1.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.31.mlps.0.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.31.mlps.0.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.31.mlps.0.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.31.mlps.1.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.31.mlps.1.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.31.mlps.1.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.32.mlp.switch_mlp.gate_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.32.mlp.switch_mlp.up_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.32.mlp.switch_mlp.down_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.32.mlp.router.classifier": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.32.self_attn.0.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.32.self_attn.0.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.32.self_attn.0.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.32.self_attn.0.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.32.self_attn.0.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.32.self_attn.0.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.32.self_attn.0.indexer.wq_b": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.32.self_attn.0.indexer.wk": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.32.self_attn.0.indexer.weights_proj": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.32.self_attn.1.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.32.self_attn.1.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.32.self_attn.1.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.32.self_attn.1.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.32.self_attn.1.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.32.self_attn.1.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.32.mlps.0.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.32.mlps.0.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.32.mlps.0.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.32.mlps.1.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.32.mlps.1.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.32.mlps.1.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.33.mlp.switch_mlp.gate_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.33.mlp.switch_mlp.up_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.33.mlp.switch_mlp.down_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.33.mlp.router.classifier": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.33.self_attn.0.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.33.self_attn.0.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.33.self_attn.0.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.33.self_attn.0.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.33.self_attn.0.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.33.self_attn.0.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.33.self_attn.0.indexer.wq_b": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.33.self_attn.0.indexer.wk": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.33.self_attn.0.indexer.weights_proj": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.33.self_attn.1.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.33.self_attn.1.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.33.self_attn.1.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.33.self_attn.1.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.33.self_attn.1.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.33.self_attn.1.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.33.mlps.0.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.33.mlps.0.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.33.mlps.0.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.33.mlps.1.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.33.mlps.1.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.33.mlps.1.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.34.mlp.switch_mlp.gate_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.34.mlp.switch_mlp.up_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.34.mlp.switch_mlp.down_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.34.mlp.router.classifier": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.34.self_attn.0.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.34.self_attn.0.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.34.self_attn.0.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.34.self_attn.0.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.34.self_attn.0.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.34.self_attn.0.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.34.self_attn.0.indexer.wq_b": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.34.self_attn.0.indexer.wk": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.34.self_attn.0.indexer.weights_proj": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.34.self_attn.1.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.34.self_attn.1.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.34.self_attn.1.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.34.self_attn.1.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.34.self_attn.1.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.34.self_attn.1.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.34.mlps.0.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.34.mlps.0.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.34.mlps.0.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.34.mlps.1.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.34.mlps.1.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.34.mlps.1.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.35.mlp.switch_mlp.gate_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.35.mlp.switch_mlp.up_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.35.mlp.switch_mlp.down_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.35.mlp.router.classifier": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.35.self_attn.0.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.35.self_attn.0.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.35.self_attn.0.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.35.self_attn.0.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.35.self_attn.0.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.35.self_attn.0.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.35.self_attn.0.indexer.wq_b": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.35.self_attn.0.indexer.wk": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.35.self_attn.0.indexer.weights_proj": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.35.self_attn.1.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.35.self_attn.1.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.35.self_attn.1.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.35.self_attn.1.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.35.self_attn.1.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.35.self_attn.1.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.35.mlps.0.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.35.mlps.0.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.35.mlps.0.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.35.mlps.1.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.35.mlps.1.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.35.mlps.1.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.36.mlp.switch_mlp.gate_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.36.mlp.switch_mlp.up_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.36.mlp.switch_mlp.down_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.36.mlp.router.classifier": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.36.self_attn.0.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.36.self_attn.0.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.36.self_attn.0.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.36.self_attn.0.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.36.self_attn.0.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.36.self_attn.0.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.36.self_attn.0.indexer.wq_b": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.36.self_attn.0.indexer.wk": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.36.self_attn.0.indexer.weights_proj": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.36.self_attn.1.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.36.self_attn.1.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.36.self_attn.1.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.36.self_attn.1.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.36.self_attn.1.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.36.self_attn.1.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.36.mlps.0.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.36.mlps.0.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.36.mlps.0.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.36.mlps.1.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.36.mlps.1.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.36.mlps.1.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.37.mlp.switch_mlp.gate_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.37.mlp.switch_mlp.up_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.37.mlp.switch_mlp.down_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.37.mlp.router.classifier": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.37.self_attn.0.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.37.self_attn.0.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.37.self_attn.0.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.37.self_attn.0.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.37.self_attn.0.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.37.self_attn.0.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.37.self_attn.0.indexer.wq_b": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.37.self_attn.0.indexer.wk": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.37.self_attn.0.indexer.weights_proj": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.37.self_attn.1.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.37.self_attn.1.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.37.self_attn.1.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.37.self_attn.1.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.37.self_attn.1.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.37.self_attn.1.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.37.mlps.0.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.37.mlps.0.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.37.mlps.0.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.37.mlps.1.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.37.mlps.1.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.37.mlps.1.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "lm_head": {
+ "group_size": 64,
+ "bits": 6
+ }
+ },
+ "quantization_config": {
+ "group_size": 64,
+ "bits": 3,
+ "mode": "affine",
+ "model.ngram_embeddings.word_embeddings": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.ngram_embeddings.embedders.0": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.ngram_embeddings.embedders.1": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.ngram_embeddings.embedders.2": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.ngram_embeddings.embedders.3": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.ngram_embeddings.embedders.4": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.ngram_embeddings.embedders.5": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.ngram_embeddings.embedders.6": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.ngram_embeddings.embedders.7": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.ngram_embeddings.embedders.8": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.ngram_embeddings.embedders.9": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.ngram_embeddings.embedders.10": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.ngram_embeddings.embedders.11": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.ngram_embeddings.embedders.12": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.ngram_embeddings.embedders.13": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.ngram_embeddings.embedders.14": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.ngram_embeddings.embedders.15": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.ngram_embeddings.post_projs.0": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.ngram_embeddings.post_projs.1": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.ngram_embeddings.post_projs.2": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.ngram_embeddings.post_projs.3": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.ngram_embeddings.post_projs.4": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.ngram_embeddings.post_projs.5": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.ngram_embeddings.post_projs.6": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.ngram_embeddings.post_projs.7": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.ngram_embeddings.post_projs.8": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.ngram_embeddings.post_projs.9": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.ngram_embeddings.post_projs.10": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.ngram_embeddings.post_projs.11": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.ngram_embeddings.post_projs.12": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.ngram_embeddings.post_projs.13": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.ngram_embeddings.post_projs.14": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.ngram_embeddings.post_projs.15": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.0.mlp.switch_mlp.gate_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.0.mlp.switch_mlp.up_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.0.mlp.switch_mlp.down_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.0.mlp.router.classifier": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.0.self_attn.0.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.0.self_attn.0.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.0.self_attn.0.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.0.self_attn.0.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.0.self_attn.0.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.0.self_attn.0.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.0.self_attn.0.indexer.wq_b": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.0.self_attn.0.indexer.wk": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.0.self_attn.0.indexer.weights_proj": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.0.self_attn.1.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.0.self_attn.1.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.0.self_attn.1.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.0.self_attn.1.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.0.self_attn.1.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.0.self_attn.1.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.0.mlps.0.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.0.mlps.0.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.0.mlps.0.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.0.mlps.1.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.0.mlps.1.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.0.mlps.1.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.1.mlp.switch_mlp.gate_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.1.mlp.switch_mlp.up_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.1.mlp.switch_mlp.down_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.1.mlp.router.classifier": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.1.self_attn.0.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.1.self_attn.0.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.1.self_attn.0.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.1.self_attn.0.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.1.self_attn.0.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.1.self_attn.0.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.1.self_attn.0.indexer.wq_b": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.1.self_attn.0.indexer.wk": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.1.self_attn.0.indexer.weights_proj": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.1.self_attn.1.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.1.self_attn.1.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.1.self_attn.1.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.1.self_attn.1.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.1.self_attn.1.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.1.self_attn.1.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.1.mlps.0.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.1.mlps.0.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.1.mlps.0.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.1.mlps.1.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.1.mlps.1.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.1.mlps.1.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.2.mlp.switch_mlp.gate_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.2.mlp.switch_mlp.up_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.2.mlp.switch_mlp.down_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.2.mlp.router.classifier": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.2.self_attn.0.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.2.self_attn.0.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.2.self_attn.0.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.2.self_attn.0.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.2.self_attn.0.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.2.self_attn.0.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.2.self_attn.0.indexer.wq_b": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.2.self_attn.0.indexer.wk": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.2.self_attn.0.indexer.weights_proj": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.2.self_attn.1.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.2.self_attn.1.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.2.self_attn.1.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.2.self_attn.1.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.2.self_attn.1.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.2.self_attn.1.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.2.mlps.0.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.2.mlps.0.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.2.mlps.0.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.2.mlps.1.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.2.mlps.1.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.2.mlps.1.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.3.mlp.switch_mlp.gate_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.3.mlp.switch_mlp.up_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.3.mlp.switch_mlp.down_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.3.mlp.router.classifier": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.3.self_attn.0.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.3.self_attn.0.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.3.self_attn.0.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.3.self_attn.0.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.3.self_attn.0.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.3.self_attn.0.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.3.self_attn.0.indexer.wq_b": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.3.self_attn.0.indexer.wk": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.3.self_attn.0.indexer.weights_proj": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.3.self_attn.1.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.3.self_attn.1.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.3.self_attn.1.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.3.self_attn.1.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.3.self_attn.1.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.3.self_attn.1.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.3.mlps.0.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.3.mlps.0.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.3.mlps.0.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.3.mlps.1.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.3.mlps.1.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.3.mlps.1.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.4.mlp.switch_mlp.gate_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.4.mlp.switch_mlp.up_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.4.mlp.switch_mlp.down_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.4.mlp.router.classifier": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.4.self_attn.0.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.4.self_attn.0.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.4.self_attn.0.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.4.self_attn.0.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.4.self_attn.0.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.4.self_attn.0.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.4.self_attn.0.indexer.wq_b": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.4.self_attn.0.indexer.wk": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.4.self_attn.0.indexer.weights_proj": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.4.self_attn.1.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.4.self_attn.1.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.4.self_attn.1.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.4.self_attn.1.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.4.self_attn.1.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.4.self_attn.1.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.4.mlps.0.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.4.mlps.0.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.4.mlps.0.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.4.mlps.1.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.4.mlps.1.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.4.mlps.1.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.5.mlp.switch_mlp.gate_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.5.mlp.switch_mlp.up_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.5.mlp.switch_mlp.down_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.5.mlp.router.classifier": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.5.self_attn.0.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.5.self_attn.0.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.5.self_attn.0.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.5.self_attn.0.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.5.self_attn.0.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.5.self_attn.0.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.5.self_attn.0.indexer.wq_b": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.5.self_attn.0.indexer.wk": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.5.self_attn.0.indexer.weights_proj": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.5.self_attn.1.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.5.self_attn.1.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.5.self_attn.1.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.5.self_attn.1.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.5.self_attn.1.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.5.self_attn.1.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.5.mlps.0.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.5.mlps.0.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.5.mlps.0.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.5.mlps.1.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.5.mlps.1.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.5.mlps.1.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.6.mlp.switch_mlp.gate_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.6.mlp.switch_mlp.up_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.6.mlp.switch_mlp.down_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.6.mlp.router.classifier": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.6.self_attn.0.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.6.self_attn.0.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.6.self_attn.0.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.6.self_attn.0.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.6.self_attn.0.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.6.self_attn.0.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.6.self_attn.0.indexer.wq_b": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.6.self_attn.0.indexer.wk": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.6.self_attn.0.indexer.weights_proj": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.6.self_attn.1.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.6.self_attn.1.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.6.self_attn.1.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.6.self_attn.1.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.6.self_attn.1.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.6.self_attn.1.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.6.mlps.0.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.6.mlps.0.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.6.mlps.0.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.6.mlps.1.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.6.mlps.1.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.6.mlps.1.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.7.mlp.switch_mlp.gate_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.7.mlp.switch_mlp.up_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.7.mlp.switch_mlp.down_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.7.mlp.router.classifier": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.7.self_attn.0.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.7.self_attn.0.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.7.self_attn.0.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.7.self_attn.0.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.7.self_attn.0.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.7.self_attn.0.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.7.self_attn.0.indexer.wq_b": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.7.self_attn.0.indexer.wk": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.7.self_attn.0.indexer.weights_proj": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.7.self_attn.1.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.7.self_attn.1.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.7.self_attn.1.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.7.self_attn.1.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.7.self_attn.1.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.7.self_attn.1.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.7.mlps.0.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.7.mlps.0.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.7.mlps.0.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.7.mlps.1.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.7.mlps.1.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.7.mlps.1.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.8.mlp.switch_mlp.gate_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.8.mlp.switch_mlp.up_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.8.mlp.switch_mlp.down_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.8.mlp.router.classifier": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.8.self_attn.0.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.8.self_attn.0.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.8.self_attn.0.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.8.self_attn.0.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.8.self_attn.0.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.8.self_attn.0.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.8.self_attn.0.indexer.wq_b": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.8.self_attn.0.indexer.wk": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.8.self_attn.0.indexer.weights_proj": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.8.self_attn.1.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.8.self_attn.1.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.8.self_attn.1.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.8.self_attn.1.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.8.self_attn.1.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.8.self_attn.1.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.8.mlps.0.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.8.mlps.0.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.8.mlps.0.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.8.mlps.1.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.8.mlps.1.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.8.mlps.1.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.9.mlp.switch_mlp.gate_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.9.mlp.switch_mlp.up_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.9.mlp.switch_mlp.down_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.9.mlp.router.classifier": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.9.self_attn.0.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.9.self_attn.0.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.9.self_attn.0.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.9.self_attn.0.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.9.self_attn.0.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.9.self_attn.0.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.9.self_attn.0.indexer.wq_b": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.9.self_attn.0.indexer.wk": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.9.self_attn.0.indexer.weights_proj": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.9.self_attn.1.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.9.self_attn.1.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.9.self_attn.1.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.9.self_attn.1.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.9.self_attn.1.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.9.self_attn.1.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.9.mlps.0.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.9.mlps.0.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.9.mlps.0.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.9.mlps.1.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.9.mlps.1.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.9.mlps.1.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.10.mlp.switch_mlp.gate_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.10.mlp.switch_mlp.up_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.10.mlp.switch_mlp.down_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.10.mlp.router.classifier": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.10.self_attn.0.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.10.self_attn.0.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.10.self_attn.0.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.10.self_attn.0.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.10.self_attn.0.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.10.self_attn.0.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.10.self_attn.0.indexer.wq_b": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.10.self_attn.0.indexer.wk": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.10.self_attn.0.indexer.weights_proj": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.10.self_attn.1.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.10.self_attn.1.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.10.self_attn.1.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.10.self_attn.1.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.10.self_attn.1.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.10.self_attn.1.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.10.mlps.0.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.10.mlps.0.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.10.mlps.0.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.10.mlps.1.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.10.mlps.1.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.10.mlps.1.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.11.mlp.switch_mlp.gate_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.11.mlp.switch_mlp.up_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.11.mlp.switch_mlp.down_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.11.mlp.router.classifier": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.11.self_attn.0.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.11.self_attn.0.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.11.self_attn.0.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.11.self_attn.0.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.11.self_attn.0.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.11.self_attn.0.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.11.self_attn.0.indexer.wq_b": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.11.self_attn.0.indexer.wk": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.11.self_attn.0.indexer.weights_proj": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.11.self_attn.1.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.11.self_attn.1.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.11.self_attn.1.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.11.self_attn.1.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.11.self_attn.1.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.11.self_attn.1.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.11.mlps.0.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.11.mlps.0.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.11.mlps.0.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.11.mlps.1.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.11.mlps.1.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.11.mlps.1.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.12.mlp.switch_mlp.gate_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.12.mlp.switch_mlp.up_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.12.mlp.switch_mlp.down_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.12.mlp.router.classifier": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.12.self_attn.0.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.12.self_attn.0.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.12.self_attn.0.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.12.self_attn.0.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.12.self_attn.0.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.12.self_attn.0.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.12.self_attn.0.indexer.wq_b": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.12.self_attn.0.indexer.wk": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.12.self_attn.0.indexer.weights_proj": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.12.self_attn.1.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.12.self_attn.1.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.12.self_attn.1.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.12.self_attn.1.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.12.self_attn.1.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.12.self_attn.1.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.12.mlps.0.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.12.mlps.0.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.12.mlps.0.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.12.mlps.1.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.12.mlps.1.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.12.mlps.1.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.13.mlp.switch_mlp.gate_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.13.mlp.switch_mlp.up_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.13.mlp.switch_mlp.down_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.13.mlp.router.classifier": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.13.self_attn.0.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.13.self_attn.0.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.13.self_attn.0.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.13.self_attn.0.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.13.self_attn.0.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.13.self_attn.0.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.13.self_attn.0.indexer.wq_b": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.13.self_attn.0.indexer.wk": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.13.self_attn.0.indexer.weights_proj": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.13.self_attn.1.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.13.self_attn.1.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.13.self_attn.1.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.13.self_attn.1.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.13.self_attn.1.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.13.self_attn.1.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.13.mlps.0.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.13.mlps.0.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.13.mlps.0.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.13.mlps.1.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.13.mlps.1.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.13.mlps.1.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.14.mlp.switch_mlp.gate_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.14.mlp.switch_mlp.up_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.14.mlp.switch_mlp.down_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.14.mlp.router.classifier": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.14.self_attn.0.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.14.self_attn.0.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.14.self_attn.0.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.14.self_attn.0.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.14.self_attn.0.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.14.self_attn.0.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.14.self_attn.0.indexer.wq_b": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.14.self_attn.0.indexer.wk": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.14.self_attn.0.indexer.weights_proj": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.14.self_attn.1.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.14.self_attn.1.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.14.self_attn.1.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.14.self_attn.1.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.14.self_attn.1.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.14.self_attn.1.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.14.mlps.0.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.14.mlps.0.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.14.mlps.0.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.14.mlps.1.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.14.mlps.1.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.14.mlps.1.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.15.mlp.switch_mlp.gate_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.15.mlp.switch_mlp.up_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.15.mlp.switch_mlp.down_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.15.mlp.router.classifier": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.15.self_attn.0.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.15.self_attn.0.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.15.self_attn.0.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.15.self_attn.0.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.15.self_attn.0.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.15.self_attn.0.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.15.self_attn.0.indexer.wq_b": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.15.self_attn.0.indexer.wk": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.15.self_attn.0.indexer.weights_proj": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.15.self_attn.1.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.15.self_attn.1.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.15.self_attn.1.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.15.self_attn.1.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.15.self_attn.1.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.15.self_attn.1.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.15.mlps.0.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.15.mlps.0.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.15.mlps.0.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.15.mlps.1.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.15.mlps.1.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.15.mlps.1.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.16.mlp.switch_mlp.gate_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.16.mlp.switch_mlp.up_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.16.mlp.switch_mlp.down_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.16.mlp.router.classifier": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.16.self_attn.0.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.16.self_attn.0.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.16.self_attn.0.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.16.self_attn.0.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.16.self_attn.0.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.16.self_attn.0.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.16.self_attn.0.indexer.wq_b": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.16.self_attn.0.indexer.wk": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.16.self_attn.0.indexer.weights_proj": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.16.self_attn.1.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.16.self_attn.1.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.16.self_attn.1.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.16.self_attn.1.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.16.self_attn.1.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.16.self_attn.1.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.16.mlps.0.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.16.mlps.0.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.16.mlps.0.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.16.mlps.1.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.16.mlps.1.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.16.mlps.1.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.17.mlp.switch_mlp.gate_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.17.mlp.switch_mlp.up_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.17.mlp.switch_mlp.down_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.17.mlp.router.classifier": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.17.self_attn.0.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.17.self_attn.0.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.17.self_attn.0.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.17.self_attn.0.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.17.self_attn.0.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.17.self_attn.0.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.17.self_attn.0.indexer.wq_b": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.17.self_attn.0.indexer.wk": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.17.self_attn.0.indexer.weights_proj": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.17.self_attn.1.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.17.self_attn.1.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.17.self_attn.1.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.17.self_attn.1.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.17.self_attn.1.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.17.self_attn.1.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.17.mlps.0.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.17.mlps.0.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.17.mlps.0.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.17.mlps.1.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.17.mlps.1.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.17.mlps.1.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.18.mlp.switch_mlp.gate_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.18.mlp.switch_mlp.up_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.18.mlp.switch_mlp.down_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.18.mlp.router.classifier": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.18.self_attn.0.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.18.self_attn.0.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.18.self_attn.0.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.18.self_attn.0.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.18.self_attn.0.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.18.self_attn.0.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.18.self_attn.0.indexer.wq_b": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.18.self_attn.0.indexer.wk": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.18.self_attn.0.indexer.weights_proj": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.18.self_attn.1.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.18.self_attn.1.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.18.self_attn.1.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.18.self_attn.1.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.18.self_attn.1.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.18.self_attn.1.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.18.mlps.0.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.18.mlps.0.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.18.mlps.0.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.18.mlps.1.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.18.mlps.1.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.18.mlps.1.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.19.mlp.switch_mlp.gate_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.19.mlp.switch_mlp.up_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.19.mlp.switch_mlp.down_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.19.mlp.router.classifier": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.19.self_attn.0.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.19.self_attn.0.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.19.self_attn.0.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.19.self_attn.0.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.19.self_attn.0.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.19.self_attn.0.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.19.self_attn.0.indexer.wq_b": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.19.self_attn.0.indexer.wk": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.19.self_attn.0.indexer.weights_proj": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.19.self_attn.1.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.19.self_attn.1.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.19.self_attn.1.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.19.self_attn.1.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.19.self_attn.1.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.19.self_attn.1.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.19.mlps.0.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.19.mlps.0.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.19.mlps.0.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.19.mlps.1.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.19.mlps.1.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.19.mlps.1.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.20.mlp.switch_mlp.gate_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.20.mlp.switch_mlp.up_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.20.mlp.switch_mlp.down_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.20.mlp.router.classifier": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.20.self_attn.0.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.20.self_attn.0.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.20.self_attn.0.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.20.self_attn.0.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.20.self_attn.0.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.20.self_attn.0.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.20.self_attn.0.indexer.wq_b": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.20.self_attn.0.indexer.wk": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.20.self_attn.0.indexer.weights_proj": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.20.self_attn.1.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.20.self_attn.1.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.20.self_attn.1.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.20.self_attn.1.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.20.self_attn.1.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.20.self_attn.1.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.20.mlps.0.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.20.mlps.0.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.20.mlps.0.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.20.mlps.1.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.20.mlps.1.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.20.mlps.1.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.21.mlp.switch_mlp.gate_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.21.mlp.switch_mlp.up_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.21.mlp.switch_mlp.down_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.21.mlp.router.classifier": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.21.self_attn.0.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.21.self_attn.0.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.21.self_attn.0.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.21.self_attn.0.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.21.self_attn.0.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.21.self_attn.0.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.21.self_attn.0.indexer.wq_b": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.21.self_attn.0.indexer.wk": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.21.self_attn.0.indexer.weights_proj": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.21.self_attn.1.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.21.self_attn.1.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.21.self_attn.1.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.21.self_attn.1.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.21.self_attn.1.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.21.self_attn.1.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.21.mlps.0.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.21.mlps.0.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.21.mlps.0.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.21.mlps.1.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.21.mlps.1.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.21.mlps.1.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.22.mlp.switch_mlp.gate_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.22.mlp.switch_mlp.up_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.22.mlp.switch_mlp.down_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.22.mlp.router.classifier": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.22.self_attn.0.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.22.self_attn.0.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.22.self_attn.0.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.22.self_attn.0.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.22.self_attn.0.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.22.self_attn.0.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.22.self_attn.0.indexer.wq_b": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.22.self_attn.0.indexer.wk": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.22.self_attn.0.indexer.weights_proj": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.22.self_attn.1.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.22.self_attn.1.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.22.self_attn.1.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.22.self_attn.1.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.22.self_attn.1.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.22.self_attn.1.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.22.mlps.0.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.22.mlps.0.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.22.mlps.0.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.22.mlps.1.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.22.mlps.1.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.22.mlps.1.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.23.mlp.switch_mlp.gate_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.23.mlp.switch_mlp.up_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.23.mlp.switch_mlp.down_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.23.mlp.router.classifier": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.23.self_attn.0.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.23.self_attn.0.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.23.self_attn.0.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.23.self_attn.0.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.23.self_attn.0.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.23.self_attn.0.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.23.self_attn.0.indexer.wq_b": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.23.self_attn.0.indexer.wk": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.23.self_attn.0.indexer.weights_proj": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.23.self_attn.1.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.23.self_attn.1.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.23.self_attn.1.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.23.self_attn.1.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.23.self_attn.1.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.23.self_attn.1.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.23.mlps.0.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.23.mlps.0.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.23.mlps.0.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.23.mlps.1.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.23.mlps.1.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.23.mlps.1.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.24.mlp.switch_mlp.gate_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.24.mlp.switch_mlp.up_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.24.mlp.switch_mlp.down_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.24.mlp.router.classifier": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.24.self_attn.0.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.24.self_attn.0.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.24.self_attn.0.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.24.self_attn.0.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.24.self_attn.0.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.24.self_attn.0.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.24.self_attn.0.indexer.wq_b": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.24.self_attn.0.indexer.wk": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.24.self_attn.0.indexer.weights_proj": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.24.self_attn.1.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.24.self_attn.1.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.24.self_attn.1.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.24.self_attn.1.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.24.self_attn.1.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.24.self_attn.1.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.24.mlps.0.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.24.mlps.0.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.24.mlps.0.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.24.mlps.1.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.24.mlps.1.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.24.mlps.1.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.25.mlp.switch_mlp.gate_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.25.mlp.switch_mlp.up_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.25.mlp.switch_mlp.down_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.25.mlp.router.classifier": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.25.self_attn.0.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.25.self_attn.0.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.25.self_attn.0.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.25.self_attn.0.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.25.self_attn.0.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.25.self_attn.0.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.25.self_attn.0.indexer.wq_b": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.25.self_attn.0.indexer.wk": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.25.self_attn.0.indexer.weights_proj": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.25.self_attn.1.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.25.self_attn.1.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.25.self_attn.1.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.25.self_attn.1.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.25.self_attn.1.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.25.self_attn.1.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.25.mlps.0.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.25.mlps.0.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.25.mlps.0.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.25.mlps.1.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.25.mlps.1.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.25.mlps.1.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.26.mlp.switch_mlp.gate_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.26.mlp.switch_mlp.up_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.26.mlp.switch_mlp.down_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.26.mlp.router.classifier": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.26.self_attn.0.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.26.self_attn.0.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.26.self_attn.0.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.26.self_attn.0.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.26.self_attn.0.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.26.self_attn.0.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.26.self_attn.0.indexer.wq_b": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.26.self_attn.0.indexer.wk": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.26.self_attn.0.indexer.weights_proj": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.26.self_attn.1.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.26.self_attn.1.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.26.self_attn.1.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.26.self_attn.1.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.26.self_attn.1.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.26.self_attn.1.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.26.mlps.0.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.26.mlps.0.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.26.mlps.0.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.26.mlps.1.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.26.mlps.1.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.26.mlps.1.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.27.mlp.switch_mlp.gate_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.27.mlp.switch_mlp.up_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.27.mlp.switch_mlp.down_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.27.mlp.router.classifier": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.27.self_attn.0.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.27.self_attn.0.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.27.self_attn.0.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.27.self_attn.0.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.27.self_attn.0.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.27.self_attn.0.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.27.self_attn.0.indexer.wq_b": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.27.self_attn.0.indexer.wk": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.27.self_attn.0.indexer.weights_proj": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.27.self_attn.1.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.27.self_attn.1.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.27.self_attn.1.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.27.self_attn.1.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.27.self_attn.1.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.27.self_attn.1.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.27.mlps.0.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.27.mlps.0.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.27.mlps.0.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.27.mlps.1.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.27.mlps.1.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.27.mlps.1.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.28.mlp.switch_mlp.gate_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.28.mlp.switch_mlp.up_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.28.mlp.switch_mlp.down_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.28.mlp.router.classifier": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.28.self_attn.0.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.28.self_attn.0.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.28.self_attn.0.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.28.self_attn.0.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.28.self_attn.0.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.28.self_attn.0.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.28.self_attn.0.indexer.wq_b": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.28.self_attn.0.indexer.wk": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.28.self_attn.0.indexer.weights_proj": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.28.self_attn.1.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.28.self_attn.1.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.28.self_attn.1.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.28.self_attn.1.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.28.self_attn.1.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.28.self_attn.1.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.28.mlps.0.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.28.mlps.0.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.28.mlps.0.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.28.mlps.1.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.28.mlps.1.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.28.mlps.1.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.29.mlp.switch_mlp.gate_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.29.mlp.switch_mlp.up_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.29.mlp.switch_mlp.down_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.29.mlp.router.classifier": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.29.self_attn.0.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.29.self_attn.0.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.29.self_attn.0.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.29.self_attn.0.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.29.self_attn.0.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.29.self_attn.0.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.29.self_attn.0.indexer.wq_b": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.29.self_attn.0.indexer.wk": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.29.self_attn.0.indexer.weights_proj": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.29.self_attn.1.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.29.self_attn.1.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.29.self_attn.1.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.29.self_attn.1.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.29.self_attn.1.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.29.self_attn.1.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.29.mlps.0.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.29.mlps.0.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.29.mlps.0.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.29.mlps.1.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.29.mlps.1.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.29.mlps.1.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.30.mlp.switch_mlp.gate_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.30.mlp.switch_mlp.up_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.30.mlp.switch_mlp.down_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.30.mlp.router.classifier": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.30.self_attn.0.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.30.self_attn.0.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.30.self_attn.0.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.30.self_attn.0.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.30.self_attn.0.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.30.self_attn.0.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.30.self_attn.0.indexer.wq_b": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.30.self_attn.0.indexer.wk": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.30.self_attn.0.indexer.weights_proj": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.30.self_attn.1.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.30.self_attn.1.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.30.self_attn.1.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.30.self_attn.1.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.30.self_attn.1.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.30.self_attn.1.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.30.mlps.0.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.30.mlps.0.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.30.mlps.0.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.30.mlps.1.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.30.mlps.1.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.30.mlps.1.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.31.mlp.switch_mlp.gate_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.31.mlp.switch_mlp.up_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.31.mlp.switch_mlp.down_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.31.mlp.router.classifier": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.31.self_attn.0.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.31.self_attn.0.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.31.self_attn.0.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.31.self_attn.0.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.31.self_attn.0.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.31.self_attn.0.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.31.self_attn.0.indexer.wq_b": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.31.self_attn.0.indexer.wk": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.31.self_attn.0.indexer.weights_proj": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.31.self_attn.1.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.31.self_attn.1.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.31.self_attn.1.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.31.self_attn.1.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.31.self_attn.1.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.31.self_attn.1.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.31.mlps.0.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.31.mlps.0.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.31.mlps.0.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.31.mlps.1.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.31.mlps.1.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.31.mlps.1.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.32.mlp.switch_mlp.gate_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.32.mlp.switch_mlp.up_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.32.mlp.switch_mlp.down_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.32.mlp.router.classifier": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.32.self_attn.0.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.32.self_attn.0.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.32.self_attn.0.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.32.self_attn.0.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.32.self_attn.0.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.32.self_attn.0.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.32.self_attn.0.indexer.wq_b": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.32.self_attn.0.indexer.wk": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.32.self_attn.0.indexer.weights_proj": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.32.self_attn.1.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.32.self_attn.1.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.32.self_attn.1.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.32.self_attn.1.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.32.self_attn.1.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.32.self_attn.1.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.32.mlps.0.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.32.mlps.0.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.32.mlps.0.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.32.mlps.1.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.32.mlps.1.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.32.mlps.1.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.33.mlp.switch_mlp.gate_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.33.mlp.switch_mlp.up_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.33.mlp.switch_mlp.down_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.33.mlp.router.classifier": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.33.self_attn.0.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.33.self_attn.0.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.33.self_attn.0.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.33.self_attn.0.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.33.self_attn.0.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.33.self_attn.0.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.33.self_attn.0.indexer.wq_b": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.33.self_attn.0.indexer.wk": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.33.self_attn.0.indexer.weights_proj": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.33.self_attn.1.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.33.self_attn.1.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.33.self_attn.1.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.33.self_attn.1.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.33.self_attn.1.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.33.self_attn.1.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.33.mlps.0.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.33.mlps.0.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.33.mlps.0.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.33.mlps.1.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.33.mlps.1.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.33.mlps.1.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.34.mlp.switch_mlp.gate_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.34.mlp.switch_mlp.up_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.34.mlp.switch_mlp.down_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.34.mlp.router.classifier": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.34.self_attn.0.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.34.self_attn.0.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.34.self_attn.0.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.34.self_attn.0.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.34.self_attn.0.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.34.self_attn.0.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.34.self_attn.0.indexer.wq_b": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.34.self_attn.0.indexer.wk": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.34.self_attn.0.indexer.weights_proj": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.34.self_attn.1.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.34.self_attn.1.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.34.self_attn.1.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.34.self_attn.1.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.34.self_attn.1.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.34.self_attn.1.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.34.mlps.0.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.34.mlps.0.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.34.mlps.0.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.34.mlps.1.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.34.mlps.1.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.34.mlps.1.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.35.mlp.switch_mlp.gate_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.35.mlp.switch_mlp.up_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.35.mlp.switch_mlp.down_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.35.mlp.router.classifier": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.35.self_attn.0.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.35.self_attn.0.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.35.self_attn.0.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.35.self_attn.0.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.35.self_attn.0.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.35.self_attn.0.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.35.self_attn.0.indexer.wq_b": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.35.self_attn.0.indexer.wk": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.35.self_attn.0.indexer.weights_proj": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.35.self_attn.1.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.35.self_attn.1.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.35.self_attn.1.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.35.self_attn.1.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.35.self_attn.1.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.35.self_attn.1.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.35.mlps.0.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.35.mlps.0.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.35.mlps.0.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.35.mlps.1.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.35.mlps.1.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.35.mlps.1.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.36.mlp.switch_mlp.gate_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.36.mlp.switch_mlp.up_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.36.mlp.switch_mlp.down_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.36.mlp.router.classifier": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.36.self_attn.0.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.36.self_attn.0.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.36.self_attn.0.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.36.self_attn.0.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.36.self_attn.0.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.36.self_attn.0.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.36.self_attn.0.indexer.wq_b": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.36.self_attn.0.indexer.wk": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.36.self_attn.0.indexer.weights_proj": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.36.self_attn.1.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.36.self_attn.1.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.36.self_attn.1.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.36.self_attn.1.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.36.self_attn.1.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.36.self_attn.1.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.36.mlps.0.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.36.mlps.0.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.36.mlps.0.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.36.mlps.1.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.36.mlps.1.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.36.mlps.1.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.37.mlp.switch_mlp.gate_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.37.mlp.switch_mlp.up_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.37.mlp.switch_mlp.down_proj": {
+ "group_size": 64,
+ "bits": 3
+ },
+ "model.layers.37.mlp.router.classifier": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.37.self_attn.0.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.37.self_attn.0.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.37.self_attn.0.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.37.self_attn.0.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.37.self_attn.0.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.37.self_attn.0.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.37.self_attn.0.indexer.wq_b": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.37.self_attn.0.indexer.wk": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.37.self_attn.0.indexer.weights_proj": {
+ "group_size": 64,
+ "bits": 8
+ },
+ "model.layers.37.self_attn.1.q_a_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.37.self_attn.1.q_b_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.37.self_attn.1.kv_a_proj_with_mqa": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.37.self_attn.1.embed_q": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.37.self_attn.1.unembed_out": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.37.self_attn.1.o_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.37.mlps.0.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.37.mlps.0.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.37.mlps.0.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.37.mlps.1.gate_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.37.mlps.1.up_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "model.layers.37.mlps.1.down_proj": {
+ "group_size": 64,
+ "bits": 6
+ },
+ "lm_head": {
+ "group_size": 64,
+ "bits": 6
+ }
+ },
+ "rms_norm_eps": 1e-05,
+ "rope_scaling": {
+ "beta_fast": 32,
+ "beta_slow": 1,
+ "factor": 120,
+ "mscale": 1,
+ "mscale_all_dim": 1,
+ "original_max_position_embeddings": 8192,
+ "rope_type": "deepseek_yarn"
+ },
+ "rope_theta": 1000000.0,
+ "routed_scaling_factor": 9,
+ "use_cache": true,
+ "use_mla": 1,
+ "v_head_dim": 128,
+ "vocab_size": 163840,
+ "zero_expert_num": 128,
+ "zero_expert_type": "identity"
+}
\ No newline at end of file
diff --git a/generation_config.json b/generation_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..d5fa01da768e00dc2b54f32a19c84e5017edd002
--- /dev/null
+++ b/generation_config.json
@@ -0,0 +1,7 @@
+{
+ "_from_model_config": true,
+ "bos_token_id": 1,
+ "eos_token_id": 2,
+ "pad_token_id": 3,
+ "transformers_version": "4.55.0"
+}
diff --git a/model-00001-of-00206.safetensors b/model-00001-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..1cb769d0ad715e87b374e55926b81ce08149eb58
--- /dev/null
+++ b/model-00001-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d92102fb7d57f3afc1abb1fc1e800dd8aeff36578f0d04313dfc2d3743de4c78
+size 4781344958
diff --git a/model-00002-of-00206.safetensors b/model-00002-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..50d6251cfb6ad6999bba7a32ccae931aeee24213
--- /dev/null
+++ b/model-00002-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:258911586aa26d14ce3a94e9cd93b7d03c9d3456acf9580a58f1db065a626f9b
+size 3690825991
diff --git a/model-00003-of-00206.safetensors b/model-00003-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..fa23ce84558996bd450d15df0ecae7719808887a
--- /dev/null
+++ b/model-00003-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:60647d08923de6ef5aeddbee301352d6f769f306c4d9103e2ac19d3967a52af4
+size 3690826439
diff --git a/model-00004-of-00206.safetensors b/model-00004-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..ae095a06ca695eb76be230523b86e4911e7e8939
--- /dev/null
+++ b/model-00004-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f69d1309e215fe8dc8b20b212dff22cc8fbbf1c594c045b5cac9b61af8f76ac6
+size 3690826887
diff --git a/model-00005-of-00206.safetensors b/model-00005-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..efbed030d71fb2f5920274471323e2807edf6b8a
--- /dev/null
+++ b/model-00005-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6f2e47b67a260d57156723c048276e828ad88625f561fbc4772d02fe42bf5338
+size 3690827335
diff --git a/model-00006-of-00206.safetensors b/model-00006-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..4ee173505b34651ccc777b00a9c97871b567b754
--- /dev/null
+++ b/model-00006-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:434c26c586da3863f8ca2f39256b094b702111fd547c49617881dfcdd99cc371
+size 3690827783
diff --git a/model-00007-of-00206.safetensors b/model-00007-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..064f258512eea2b6473ace40217a393d2eaf3826
--- /dev/null
+++ b/model-00007-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fd1675f65434f613b66340b9f52040c70107da3c38867efaa6648c2f48fec222
+size 3690828231
diff --git a/model-00008-of-00206.safetensors b/model-00008-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..edc6f46d9571cc5077f017fdc07ccbabb160f09e
--- /dev/null
+++ b/model-00008-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0d53f80922111ac94f4b678487d8b18ed74978e4106f0b073517386ca88f7ba7
+size 3690828679
diff --git a/model-00009-of-00206.safetensors b/model-00009-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..74912d5bc655f4a15296caf6e4857f83b2004aa6
--- /dev/null
+++ b/model-00009-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0aa1b0c0e3d8bdc1906ebdab4f8dc34b37d5d1720c12eeb710656c2a5be106f1
+size 3690829127
diff --git a/model-00010-of-00206.safetensors b/model-00010-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..a522c2f67d2f94d4d1e007a9285002486a608672
--- /dev/null
+++ b/model-00010-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:de3a6d7f93810038473245300b35110f7a996db89353dd68b2df939945b33f0f
+size 3690829575
diff --git a/model-00011-of-00206.safetensors b/model-00011-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..80f0149f1ca1772888c2351af18ecc59e1af4336
--- /dev/null
+++ b/model-00011-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f5f60aedf86247a7e47f1d2d27a9f1024995afb71e1d9ad6cec9e02c4a8a1235
+size 3690830026
diff --git a/model-00012-of-00206.safetensors b/model-00012-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..a132a1d18d6f048876cc364ebca63e390694f961
--- /dev/null
+++ b/model-00012-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7597a9e97862be4d19b19b3c1ff81a4a268649d7d1e43155c74d72bb2498c57e
+size 3690830474
diff --git a/model-00013-of-00206.safetensors b/model-00013-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..3561f0f575cd796d59c0714b7e7814825abebd25
--- /dev/null
+++ b/model-00013-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:345077719b1eff83770e6cfe6ce42a6b6ebca0354ebd613ad3ab31e554c21d19
+size 3690830922
diff --git a/model-00014-of-00206.safetensors b/model-00014-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..460b2de8ea13c0d53e5c5c1395e1294f6dc07b0e
--- /dev/null
+++ b/model-00014-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:16e3d2388d0e19e49846950716122ca9112ab995eeeece556bfa75e979b34741
+size 3690831370
diff --git a/model-00015-of-00206.safetensors b/model-00015-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..9d7512cddd8d29015f10f2092047ee04f73be035
--- /dev/null
+++ b/model-00015-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:76362eddbafd21a9829e8bc3b4d7e2d351f015fad438f16abfad041e077574e6
+size 3690831818
diff --git a/model-00016-of-00206.safetensors b/model-00016-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..69901e29b8a2eb4f7289681c4f6ca268b6d425c1
--- /dev/null
+++ b/model-00016-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3fbca5cd0a90ec817b3d81fd2e952c57f00f4296000076d0df33c5c15acc5c62
+size 3745363732
diff --git a/model-00017-of-00206.safetensors b/model-00017-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..615cbe08918fa45349ffb6f18ae6ac0dd72788a6
--- /dev/null
+++ b/model-00017-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e34ef72cb80fdd56bd3486d1e4efc6d916c3c4df812e6b131dd697a3f8c4f631
+size 5234491677
diff --git a/model-00018-of-00206.safetensors b/model-00018-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..89962a84254121bbce919b5c08da18f07d5d044d
--- /dev/null
+++ b/model-00018-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:74a17ad528b2143ba88fde9c841b2ae8714df2c2188c86fac22ba0845d751075
+size 5234491677
diff --git a/model-00019-of-00206.safetensors b/model-00019-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..127b2723f415c1fb75e8d04a337240ded1918e01
--- /dev/null
+++ b/model-00019-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:87c3f60b8af8c7cdbd45a26e71af942530ad295f4a97914ef797502abe78b1b8
+size 805306649
diff --git a/model-00020-of-00206.safetensors b/model-00020-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..ea20251082897bf9cae117e158b74fc53a574233
--- /dev/null
+++ b/model-00020-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9d0f21b54bba92a1cad8fab0611bcc82cdb7ff9f344a2a82267abd7e7434105c
+size 5234491676
diff --git a/model-00021-of-00206.safetensors b/model-00021-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..a4cb74afafd8afbf56c986c81c101ba67b567e30
--- /dev/null
+++ b/model-00021-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d98358d43c73a4587080ba3a3d7795c009f59246d0c678ce0eaaf6d167e6e42d
+size 1091293851
diff --git a/model-00022-of-00206.safetensors b/model-00022-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..406d6a99acca4688be2ddb6e804b07fd5af20263
--- /dev/null
+++ b/model-00022-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9ffdabb83a66ca77efe4d4121d10c3610725053a80bc9c963a9fd2f3badc4d11
+size 5234491677
diff --git a/model-00023-of-00206.safetensors b/model-00023-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..2910168abc65b5a6d8a313925c2cae6623797468
--- /dev/null
+++ b/model-00023-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2161503a0861b798e513c620f115d9187aadf6ad71e7e53eeb8ad29c97b3427d
+size 5234491677
diff --git a/model-00024-of-00206.safetensors b/model-00024-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..aa87e609d1f050d804443633f49dc8ef0cb4da30
--- /dev/null
+++ b/model-00024-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9cd60d3053c71716ad6e2b45c259c7af6501c0f77bd8d69f4ad8e178f17462e2
+size 805306649
diff --git a/model-00025-of-00206.safetensors b/model-00025-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..0f2ab333a6c7715a3a4fbe205585c280d08e9982
--- /dev/null
+++ b/model-00025-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c696e34582376e5e546a1be4f8e18a94017d0a18374fbf39d085b0f99f6ae98f
+size 5234491676
diff --git a/model-00026-of-00206.safetensors b/model-00026-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..18ac0bf4b6a73d373f618848bcfae8a65aeda5a6
--- /dev/null
+++ b/model-00026-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1b0d29f44d83e2c7ae666e0ae3b4732a8de5f1f8d92a98457fa64db4ba30a35d
+size 1091293841
diff --git a/model-00027-of-00206.safetensors b/model-00027-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..cd330de563dfcd7d90cdeedd09f3995a20238f6c
--- /dev/null
+++ b/model-00027-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dcd51638dd23a650de22b22df0e0231c3ef608f4f44eeab1d97b45e2fdd0a40e
+size 5234491677
diff --git a/model-00028-of-00206.safetensors b/model-00028-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..2e6215ce1fdbd4642279d597ee24be5d57ab301f
--- /dev/null
+++ b/model-00028-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7d25cc7a045c1416a9c8f030d165c033569cc597405d596927fb0dad58942fa3
+size 5234491677
diff --git a/model-00029-of-00206.safetensors b/model-00029-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..2f8d034913f718557e69cf149b113c79a57f50b3
--- /dev/null
+++ b/model-00029-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3207a9fc41cd05593388c94008cddaf68453d6fb1cf298f304a56072363962b4
+size 805306649
diff --git a/model-00030-of-00206.safetensors b/model-00030-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..89aacf73a087ecbaecde735f28d4ca6ba0cf2091
--- /dev/null
+++ b/model-00030-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f97a49c65466a073eb842d30478db93f3414187bab65d1be00cc0bdb11690b33
+size 5234491676
diff --git a/model-00031-of-00206.safetensors b/model-00031-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..3cb9581a80d7549e5de6298873b4f172aa24efe9
--- /dev/null
+++ b/model-00031-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0db114df9cea36449192c7a6e8eb04bb1e81d7be90450e3f5a03c44b21e13d96
+size 1091293847
diff --git a/model-00032-of-00206.safetensors b/model-00032-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..9b54a1ceb6b63a491aa83a7f8647ec8e1fcf980d
--- /dev/null
+++ b/model-00032-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2bc1676df0841af6ba3d67b86c5cae51f1b2850ff95951ad5edcc12991bf5337
+size 5234491677
diff --git a/model-00033-of-00206.safetensors b/model-00033-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..c3957eb79cec4d66bbe9e5b8562002401d475928
--- /dev/null
+++ b/model-00033-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:00076dc26d2eee360fb45b40d55496e5d7202bd513a7fcfbe4dd70a988178c23
+size 5234491677
diff --git a/model-00034-of-00206.safetensors b/model-00034-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..2cc5ec396cf130a49e4bc1041f20ccfd5c0e8f07
--- /dev/null
+++ b/model-00034-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b73aac7a360d6dac773fdd540d02811fbeb2837f76f8bd034f9d61684da4b48a
+size 805306649
diff --git a/model-00035-of-00206.safetensors b/model-00035-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..81d2316075bad89d516b924c07ce111c5725a7d0
--- /dev/null
+++ b/model-00035-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:12cc45c098878b4568be3e2cebd01d2d88c28af276658434a4b5e4b1b0cb746e
+size 5234491676
diff --git a/model-00036-of-00206.safetensors b/model-00036-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..ff005d226b0ff9f527a7cf37d573ed28bfc1f643
--- /dev/null
+++ b/model-00036-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d5e6d5a2aa722873b1d9cfa6c8b0d0330bb2a250f6a239ec42ae02cd5bd2c813
+size 1091293863
diff --git a/model-00037-of-00206.safetensors b/model-00037-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..7917e54e6face5eff4bbf15f243a665bd89d2ccd
--- /dev/null
+++ b/model-00037-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:faa2b2814dcb5eaf8608f430729ec481d633253e847dea7fb17e484036e0c419
+size 5234491677
diff --git a/model-00038-of-00206.safetensors b/model-00038-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..6c8de1f2bc29dcbf41c95688f32acf5011120c1d
--- /dev/null
+++ b/model-00038-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:602b1fd064afb5a8cd97b72cc9c1c0802c67f2bd5f1c9cb8b9c4c1ad78a9bbaa
+size 5234491677
diff --git a/model-00039-of-00206.safetensors b/model-00039-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..3db499bddddca654d25bf043cb4a639516707075
--- /dev/null
+++ b/model-00039-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:242bf90e08149a982453dec5ec171db03aaa0f19a25b7e27653817eb626d3c8d
+size 805306649
diff --git a/model-00040-of-00206.safetensors b/model-00040-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..77a4c258e8dbe9d81282b0dc7b2e01249354e094
--- /dev/null
+++ b/model-00040-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3dd4a965763b7d419108b00c2999deb869288fdcbd4e91c0f0aa654c1addb673
+size 5234491676
diff --git a/model-00041-of-00206.safetensors b/model-00041-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..d41d7f8919790773cfb9e104e04082bba5205d84
--- /dev/null
+++ b/model-00041-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e63afa2bfe9b0108a01304eb064d81dc01ba60ebfdfa7d56b57f14d7a700424d
+size 1091293881
diff --git a/model-00042-of-00206.safetensors b/model-00042-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..ccd6f735ac73a622bbdc1e3a9ecb189e856a7f0f
--- /dev/null
+++ b/model-00042-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d3d93b8bfe8144efae08fa0738aa1918f9053a305e5b116992f1a201007bfa6f
+size 5234491677
diff --git a/model-00043-of-00206.safetensors b/model-00043-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..d2680ce60c5a628e0e0dfbd0b6853ccb8d040db2
--- /dev/null
+++ b/model-00043-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a2ec74e651ca06a633b8c23fee42fb2d7e50da9a489f6c7744a52d4f09457cbd
+size 5234491677
diff --git a/model-00044-of-00206.safetensors b/model-00044-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..76f84b1035434d594ee4725ab8d6613805e7f057
--- /dev/null
+++ b/model-00044-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8f8e7b0647767acaa74c9cfcebd6c2d81f889be92333e7eb4a7fa2169615da62
+size 805306649
diff --git a/model-00045-of-00206.safetensors b/model-00045-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..b2338e81fcc65bc423d982b38f2a970104fc4471
--- /dev/null
+++ b/model-00045-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7df941301d75520277850926643daded34d0512fad1006319220a7d35e597bd8
+size 5234491676
diff --git a/model-00046-of-00206.safetensors b/model-00046-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..33ea5443790fb1d2d10ba00ed24f14fa9c0a351e
--- /dev/null
+++ b/model-00046-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b72d67805aa622535adf898e3518e232379c8a8eb44cacb6364664f64d3b6de
+size 1091293865
diff --git a/model-00047-of-00206.safetensors b/model-00047-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..d8992f51f26f695346503b81fca40aa2a6217394
--- /dev/null
+++ b/model-00047-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6e0f9e395f9b5a0c024a4dbea405f46683c830627e7341e98ef0ab683f4446e4
+size 5234491677
diff --git a/model-00048-of-00206.safetensors b/model-00048-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..d95c9cc4538abaffb5aab330d924cc8b13a5a474
--- /dev/null
+++ b/model-00048-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:db8e2dda3a5810b3a13f585fac42a721dd961fb0348f3e8ce0b8d745962d7cd1
+size 5234491677
diff --git a/model-00049-of-00206.safetensors b/model-00049-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..d52931cb044f29499e940dfeae84d5342d4f77b7
--- /dev/null
+++ b/model-00049-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:98cc6d61b6f4edf48626667489cdc9c7efbe6d5e513869226e2f15790918b061
+size 805306649
diff --git a/model-00050-of-00206.safetensors b/model-00050-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..f48d3790bc0c4c89682139366795492868ef1c67
--- /dev/null
+++ b/model-00050-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:24227190bffd66f646f39b0f1c3cb1947ed385c149238b4ea74f27eabefffdeb
+size 5234491676
diff --git a/model-00051-of-00206.safetensors b/model-00051-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..92f0299b19abc2a0f9c23912abd17e6ee313de77
--- /dev/null
+++ b/model-00051-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:efb82a5ec11c74f24da3435fb7a24fa4ae10eb1d5e6aae3f2be150f82b9a8da4
+size 1091293873
diff --git a/model-00052-of-00206.safetensors b/model-00052-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..6de54eea94603511e38d04d7e539fba15d6bf3b7
--- /dev/null
+++ b/model-00052-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:84ed3e0351478908cc85314be1d1798492c6f58647e93b6c0ffc8175bedcc68b
+size 5234491677
diff --git a/model-00053-of-00206.safetensors b/model-00053-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..455b8a0c5e10f42144734ab272e09c719fefaf44
--- /dev/null
+++ b/model-00053-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:101066fc68151f9dbd5bf71e4e233eec59168146e6cdcfbb83af5a7992126655
+size 5234491677
diff --git a/model-00054-of-00206.safetensors b/model-00054-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..60d7697c87516ff4ab5b51e2cd0f3f806434e109
--- /dev/null
+++ b/model-00054-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ed529ef4cb8b098fe3a8bc056d69c5596e3516f02243fa390b6a923a415c6470
+size 805306649
diff --git a/model-00055-of-00206.safetensors b/model-00055-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..928fe7d1af8503633ba5d4a8c37e5a6cbc15f4b4
--- /dev/null
+++ b/model-00055-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7d50d01f522f168097f059423d5789d9adaa093647d99c331f039d8ab6b08d42
+size 5234491676
diff --git a/model-00056-of-00206.safetensors b/model-00056-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..52aa8799bbf7848c4c5b94c7931c1e05e83dd9dc
--- /dev/null
+++ b/model-00056-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cff5fe1c611eecd556afd95c84570c92c03fb482f1b089a0b86389597c716585
+size 1091293871
diff --git a/model-00057-of-00206.safetensors b/model-00057-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..74b5c9d1a9599372d4566fa332b1cfc5920969a7
--- /dev/null
+++ b/model-00057-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:195eecca0232274f099fc09804742357ab746937480484216989395aa29e5bbe
+size 5234491677
diff --git a/model-00058-of-00206.safetensors b/model-00058-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..2c91e4b58edbd6f3449fac60e69ed9ed3cbd80a8
--- /dev/null
+++ b/model-00058-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7d7938eda9157508a539db6ce2051493574aeaa092bd74977f55c900a4cdf696
+size 5234491677
diff --git a/model-00059-of-00206.safetensors b/model-00059-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..2aeb9ac2561662cc0a2b8bd6c94fabb539d44c70
--- /dev/null
+++ b/model-00059-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dffa351f97189a47b86f759004164c094d94092a99cbaf8e80f6eaf35704b93c
+size 805306649
diff --git a/model-00060-of-00206.safetensors b/model-00060-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..2d8e8277866ae627ca33b54e58e0b99c1a78f4e3
--- /dev/null
+++ b/model-00060-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8b64b930a61fe4c6b3b942baa696f663e1b99709384cfcb21300c0a0d2564e6b
+size 5234491676
diff --git a/model-00061-of-00206.safetensors b/model-00061-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..68518c126a34e29d19bbb502f9642c35f8076fa9
--- /dev/null
+++ b/model-00061-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:edeaffa2f3f4dbe9d1ec74c5df8e3d321e25b0eed354466d6a26a2e53271b214
+size 1091293843
diff --git a/model-00062-of-00206.safetensors b/model-00062-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..02df8a9bb546e8465b9e32071019a448c3f21ed3
--- /dev/null
+++ b/model-00062-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d976a157968ba362aec934508ad39bc52aae7dbe704c80f228b93f37784e98cb
+size 5234491677
diff --git a/model-00063-of-00206.safetensors b/model-00063-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..4093a5f7f66098307867c2afb0c238f969b813db
--- /dev/null
+++ b/model-00063-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4f2a5c925c7d8342784c4db73db1fccfa58b9e33bc2854b505c3b2e59468b44e
+size 5234491677
diff --git a/model-00064-of-00206.safetensors b/model-00064-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..6e97a8084769d23e1c4637fcbd88b4fe8141da47
--- /dev/null
+++ b/model-00064-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:747d96e30a2aa9245a8591e2b021884379fc2a567b8b906217e83eca91cb5924
+size 805306649
diff --git a/model-00065-of-00206.safetensors b/model-00065-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..15679eedcc81247e9deb9383ee187b7f64f73c4d
--- /dev/null
+++ b/model-00065-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:506ea7680b7a1bf99a02a87addb777952deb429ef66e383fb16061e1bfe6a099
+size 5234491676
diff --git a/model-00066-of-00206.safetensors b/model-00066-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..047f2a4bd3b1de4d0092bb62bde28b819e92c6b3
--- /dev/null
+++ b/model-00066-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6286096f3de843dcd856fcf322034e12925e64fb5b25ad984bb6fb1e13f4f8ae
+size 1091293869
diff --git a/model-00067-of-00206.safetensors b/model-00067-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..913899115e9d219de267f1ec6095a241622bdb93
--- /dev/null
+++ b/model-00067-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a9b7e825231e516b74e03a671d66a0d5c49ef8bc504ef4f506d05e2da8db5a58
+size 5234491679
diff --git a/model-00068-of-00206.safetensors b/model-00068-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..27156591ff17fb12ee9063e698804f078b292628
--- /dev/null
+++ b/model-00068-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c00eba822dd009b46789f041fd492c1a42f145855229c73109dc034d179f2be0
+size 5234491679
diff --git a/model-00069-of-00206.safetensors b/model-00069-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..0f6e71c3d7d85e6f5e2d556692bcb6fdd868f167
--- /dev/null
+++ b/model-00069-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b85d5cbb3f2ea6370fe907a76810bca730d5b9f9f77ef6fdbd418c13c56ccd57
+size 805306651
diff --git a/model-00070-of-00206.safetensors b/model-00070-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..9813ee57f4c49245d690b50b154f1b38bb9a30c7
--- /dev/null
+++ b/model-00070-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4da5292b851031eb72263d8c0251e2ba21be671c14b6faa66d64f9a2c4e2002a
+size 5234491678
diff --git a/model-00071-of-00206.safetensors b/model-00071-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..b2e5a31dc4d455ae26ad1ee5d83f747c17489b94
--- /dev/null
+++ b/model-00071-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e57f3aeebbfe861615390d491f5ee4a619bd4ef7655ada9630820a19d97d8678
+size 1091293928
diff --git a/model-00072-of-00206.safetensors b/model-00072-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..b74a8ad0258ecb5f9ecf5f28dd380fcc922c04a5
--- /dev/null
+++ b/model-00072-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:56f67b1778ba8db3a279b261796beaab33559a2bbb48fc248ac82269429fc0bc
+size 5234491679
diff --git a/model-00073-of-00206.safetensors b/model-00073-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..381de332f1d3c6c4a4e99b5d27cb54dbbf34e4e9
--- /dev/null
+++ b/model-00073-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:adab9f3512eaeefa48a9e66687f3256542226c2a10aaf1ef6d0716a8e7088bd5
+size 5234491679
diff --git a/model-00074-of-00206.safetensors b/model-00074-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..5998ab46b0ee008d6a943747a60d41cefe3e616d
--- /dev/null
+++ b/model-00074-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2de4a346b39077f01e224c16581388d32e91d6b7db0b8498475ce0b4a2384154
+size 805306651
diff --git a/model-00075-of-00206.safetensors b/model-00075-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..6e81e8ef4c98aaab8b485777d572e9d6040e530a
--- /dev/null
+++ b/model-00075-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:02ff2ea8386869e9c735fd3b511b6ec6579842f52bab21c0f2cc21eab6bd2e40
+size 5234491678
diff --git a/model-00076-of-00206.safetensors b/model-00076-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..f258ff523fc8690426b8e579ba51268d4bbac633
--- /dev/null
+++ b/model-00076-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:37b09a7d7135424bfc504908e104364123e89ef63bc7cac918d5ac31274feb91
+size 1091293952
diff --git a/model-00077-of-00206.safetensors b/model-00077-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..71520a73ba5584041355d0fd1cb6a8308b2ef528
--- /dev/null
+++ b/model-00077-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cb556aa215a152fe6248a195b5bfb190f93a32feb7c91e08c746ff248627d27b
+size 5234491679
diff --git a/model-00078-of-00206.safetensors b/model-00078-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..564ee46149b5fba0fb43246984b6e464aefdc797
--- /dev/null
+++ b/model-00078-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ca324da0ede27e46f4200e1046cf59a384e159d2b8141946bef3dc7e2c583ad8
+size 5234491679
diff --git a/model-00079-of-00206.safetensors b/model-00079-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..e01cdf5a51b821e7a2481797967690e77ab12376
--- /dev/null
+++ b/model-00079-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:46d1d5202801aeccbae075b922fb96a098509a84341d41cdf33a10081be5e370
+size 805306651
diff --git a/model-00080-of-00206.safetensors b/model-00080-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..bef0df29099db182b8e90fe17008b007e807ea83
--- /dev/null
+++ b/model-00080-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:81cb5f49467737dfd961a4c80cda3bf9e032fd579a6d2c95b1e6ae2784468ec8
+size 5234491678
diff --git a/model-00081-of-00206.safetensors b/model-00081-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..50c711c31de205d13223cbdee5ff7f6884ecbae3
--- /dev/null
+++ b/model-00081-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:59a891b6eb17a4d677edc815b2f53a8b3d2459712961b1844c280e7eb1f6f0ff
+size 1091293934
diff --git a/model-00082-of-00206.safetensors b/model-00082-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..38bf5868e90a10f74ddbdd88118ccb4aac6e79cd
--- /dev/null
+++ b/model-00082-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eebd1756b7ecfed36f858b77bb84e8e63e37175ab0d0674c3df96abde32a590b
+size 5234491679
diff --git a/model-00083-of-00206.safetensors b/model-00083-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..59fd1c63c76b9b4cc248a7d51963ea68cc348de9
--- /dev/null
+++ b/model-00083-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4e6b3ab19ee87aa4e251d8409476369dc96283281d944d5af04addadbed742be
+size 5234491679
diff --git a/model-00084-of-00206.safetensors b/model-00084-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..3caacad9a6f8e0e356777e57967615dcf31b3cbe
--- /dev/null
+++ b/model-00084-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7f9af4577f0fac326d72162a2bdce5a45a327cf66a9accafb2ef48524f222a9c
+size 805306651
diff --git a/model-00085-of-00206.safetensors b/model-00085-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..1ac99685b3609f8628f23fad4324efbfdd43a236
--- /dev/null
+++ b/model-00085-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:21ae8427a808ef59f173e842431288135c2943a3481281183a734e5f5aeb1723
+size 5234491678
diff --git a/model-00086-of-00206.safetensors b/model-00086-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..20cf07d66f53a6ba8adc29e555dbf7c4ce38bbd3
--- /dev/null
+++ b/model-00086-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ba835255ee973f64bda01993ce7f095e0001c1298163268ea31d83cf0e4316dc
+size 1091293958
diff --git a/model-00087-of-00206.safetensors b/model-00087-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..f770aebc5173540940d33b07db391b4419a50859
--- /dev/null
+++ b/model-00087-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:628b17bdaf3fda1a2c5a004ec1e363fd069c27ac8e5b47157b1038353ea56639
+size 5234491679
diff --git a/model-00088-of-00206.safetensors b/model-00088-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..7f02102b6df88211224ca9e70721d128efcd5cad
--- /dev/null
+++ b/model-00088-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f304eeaab96744e3aaa455b2c9a2d016e43dedbb5f314e6d46f55bbb3c663c43
+size 5234491679
diff --git a/model-00089-of-00206.safetensors b/model-00089-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..c552df76c452f74e31660531f79ee5acf370e5c3
--- /dev/null
+++ b/model-00089-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:286710a701d7d18869e874a39bc25c09e50fef8404806e4210e66a6638f74f8e
+size 805306651
diff --git a/model-00090-of-00206.safetensors b/model-00090-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..e1de43833bf6a800e5aca3cdd8fd879ae3218b54
--- /dev/null
+++ b/model-00090-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fb8a796b024d1a6ea6011b462915beddf9f024d688cfe6c9f916b728287a9f34
+size 5234491678
diff --git a/model-00091-of-00206.safetensors b/model-00091-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..132184e859948ff0261ee793fa8e73403a6aa2bc
--- /dev/null
+++ b/model-00091-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:92dc1f550c22e4a0ee5ffaba2bf8dc9aeb1375c39152cc9f934c509a7f7f95a5
+size 1091293950
diff --git a/model-00092-of-00206.safetensors b/model-00092-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..5c35e0b495dcac883f7aecbb60911e66a4d8bde7
--- /dev/null
+++ b/model-00092-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8f1d906098c5bbdeda88ec98fb11c00a6a8cc9b430ca9b3d91d42031f2909900
+size 5234491679
diff --git a/model-00093-of-00206.safetensors b/model-00093-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..d0c99a770555ecf94d0d4710fbedc5ccff15ce97
--- /dev/null
+++ b/model-00093-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:854fc21b7b2c5f28c047ebcf0f8f8d0432586727f67e2ef23636530133c1224d
+size 5234491679
diff --git a/model-00094-of-00206.safetensors b/model-00094-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..06ebcd47c1c0bee29032c86ebc7895d28ca48836
--- /dev/null
+++ b/model-00094-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f0a1374db2e704a555a6b4fc12bd27a3522d3f0856aa720cd33c8c3ff57f5820
+size 805306651
diff --git a/model-00095-of-00206.safetensors b/model-00095-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..c61136bafb33d85efd765a91ec0e2b7f1b5a9fbe
--- /dev/null
+++ b/model-00095-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9b6c471b7f8b440fe04be9e9bde6d62879ca5dda7493fe12b1190506dd370568
+size 5234491678
diff --git a/model-00096-of-00206.safetensors b/model-00096-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..a5720a3485e6ce1ee864b7a5cd688fa016c52c7b
--- /dev/null
+++ b/model-00096-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c62f51af58bf0dec9e5fce8413030c83daee6c5d0a064e99ee29ec7db8810eb4
+size 1091293916
diff --git a/model-00097-of-00206.safetensors b/model-00097-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..d97fb33baea236f430f0ef16e317f17a30d8a9ff
--- /dev/null
+++ b/model-00097-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:55c63963c22687926f91416e1d128f0efdc7452207c6de5e6b584abbe22fb110
+size 5234491679
diff --git a/model-00098-of-00206.safetensors b/model-00098-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..20797077a23273d13286e612c2cc7e5fff1b4d5b
--- /dev/null
+++ b/model-00098-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:074a0aec389c5c8a1b34d597f2f976324c0d9485aed31af40004ad5ba55f1bb7
+size 5234491679
diff --git a/model-00099-of-00206.safetensors b/model-00099-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..7da2aabea0b14695b954c75885debae7f8841c46
--- /dev/null
+++ b/model-00099-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a2d756e82051dcb39c936bc344c1cd2107df98203da79d9f884124eecd487774
+size 805306651
diff --git a/model-00100-of-00206.safetensors b/model-00100-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..b019e100599784a871353547ab302a388f1a1544
--- /dev/null
+++ b/model-00100-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e507f5cbc519f2da5909cb57d7cf1e8dce7740d4dd9668d1e6dec34797b18448
+size 5234491678
diff --git a/model-00101-of-00206.safetensors b/model-00101-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..d7bd5bec71cb372c504a234ee096da5dae289bb4
--- /dev/null
+++ b/model-00101-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bbd51101ef9a5f2d4c21c96481ea335f117e1f8550be1760b6538d80b4bf4104
+size 1091293956
diff --git a/model-00102-of-00206.safetensors b/model-00102-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..b7e7b49129acaaa6d2946924c51ea525e562eada
--- /dev/null
+++ b/model-00102-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:556ce2d88a02e599c166bba5fe12a11a99a6b7f1b9fdc55e015bf019e4498213
+size 5234491679
diff --git a/model-00103-of-00206.safetensors b/model-00103-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..79cd38550ed9edc34d510edce154ca36251cd346
--- /dev/null
+++ b/model-00103-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a9db74d72deebb7e8fb1b2c2043b80fc4e081dbafe4711b64f5d6be3ab0f13d9
+size 5234491679
diff --git a/model-00104-of-00206.safetensors b/model-00104-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..f4d95e177647215dbd17d7eb817cee0f6824aa1c
--- /dev/null
+++ b/model-00104-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5104ee429a67a03e0ea7988822a38139cfb2cd076cc9c3b8c720555a81785b96
+size 805306651
diff --git a/model-00105-of-00206.safetensors b/model-00105-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..d21473e9bdcbd0d7b2c0a9f54b301141325ced97
--- /dev/null
+++ b/model-00105-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6f4b2ed0609b0f69a93afa4810117286e4e4ad93c1b04b761af96c58953d7475
+size 5234491678
diff --git a/model-00106-of-00206.safetensors b/model-00106-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..4fb6c951b0f37aeb01f4a13dcdeec05e4b4eda32
--- /dev/null
+++ b/model-00106-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1621357ecf60b2e45d10919806d051623e3516c58587c84ee0a65d6ec4800a3e
+size 1091293970
diff --git a/model-00107-of-00206.safetensors b/model-00107-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..bf44d2a65f0c57518eb19c0699893a47d836ed27
--- /dev/null
+++ b/model-00107-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6ed4fe2dc3944a1134cd85cde4cb4990a05183e94a8a00fd200f6fdf8d7c37db
+size 5234491679
diff --git a/model-00108-of-00206.safetensors b/model-00108-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..cd25965f719fb9ceb5b3061940a4dc5892fdcff6
--- /dev/null
+++ b/model-00108-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a454f709250cd9d5e63dd5e349ec6c66cb065536af70d81a559941e1a2550eb0
+size 5234491679
diff --git a/model-00109-of-00206.safetensors b/model-00109-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..7386dd3bd28b68abe51ea5fa2700835547acc7df
--- /dev/null
+++ b/model-00109-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6e2b60de00b1488202b00efc8e0a8f012cc6ec1be1d42c06f29c4ca7afa74924
+size 805306651
diff --git a/model-00110-of-00206.safetensors b/model-00110-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..fa9b4267b913e732df00962a6b864a7cf3a75dc5
--- /dev/null
+++ b/model-00110-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b7db1dfac968ceedcfe52e36ecafce693044fc117177da9486ec04171c8b1058
+size 5234491678
diff --git a/model-00111-of-00206.safetensors b/model-00111-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..a4381fdedcc19c6002ed7f088504e0e8c8d9ef23
--- /dev/null
+++ b/model-00111-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a8a34928a7b715b2f8dc6f054a34fa8763a62c43fbb6ff3a44d72d64a2716cfd
+size 1091293940
diff --git a/model-00112-of-00206.safetensors b/model-00112-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..5b64735a10d572f5daf0ad6877db9c8de16f3d5a
--- /dev/null
+++ b/model-00112-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:90efa3f88461277caae46452e7e1e2d7a39912c17058ce8e7dc9c3608cb33c78
+size 5234491679
diff --git a/model-00113-of-00206.safetensors b/model-00113-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..a5533bcdc4831df102fd11ad20e2b203155c078a
--- /dev/null
+++ b/model-00113-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:723c128709f85a035ad7cb77ec92a71091632395323d32fbfe3bb5660dcfc0d4
+size 5234491679
diff --git a/model-00114-of-00206.safetensors b/model-00114-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..139c36dbff78c6cd46e7ffbe68fb12e61314458c
--- /dev/null
+++ b/model-00114-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:968939d006d6b23990e1ac229031f2e7b7c1a078ad4d5b44cc11b284f2fd124c
+size 805306651
diff --git a/model-00115-of-00206.safetensors b/model-00115-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..9fe1422ab80935514dc4359fc12373f4d574292c
--- /dev/null
+++ b/model-00115-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5297ad3512b40138b0fc742fa3c2b54306e794de2759bfd58e3817a3078d3f71
+size 5234491678
diff --git a/model-00116-of-00206.safetensors b/model-00116-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..12378a9afc40ae69c8c488d97a701b6b4ae316ad
--- /dev/null
+++ b/model-00116-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e52f68ae45a18a48053a0734e21de409a78720fdb76e4018d2f17711c7ad4088
+size 1091293924
diff --git a/model-00117-of-00206.safetensors b/model-00117-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..2d5b08ee3c0a593459ae438299d50e5e85a692d8
--- /dev/null
+++ b/model-00117-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:25c15ea5a0c6b657aaa9b230446c046b07311faf191145a109e199c4fa99fca2
+size 5234491679
diff --git a/model-00118-of-00206.safetensors b/model-00118-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..8772befc84fdc8208e655c3274f52d220f07f296
--- /dev/null
+++ b/model-00118-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1bf3814423b05ad459a835c5ac0d6bc3e2e6e59c9f09c85ca26420e7530f6066
+size 5234491679
diff --git a/model-00119-of-00206.safetensors b/model-00119-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..71b2d0dbe50fd1f17ac03a083261a1b110141b12
--- /dev/null
+++ b/model-00119-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9583fff09982742adc98c33e612cc776990d8f8c2441969b7171b51d8dd65025
+size 805306651
diff --git a/model-00120-of-00206.safetensors b/model-00120-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..9b8f56b88043a26defefaed6ad04d2a412122c49
--- /dev/null
+++ b/model-00120-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:39b0a6631128dfe9de651d350171c8548dcbd14857b3713066bd042da957d663
+size 5234491678
diff --git a/model-00121-of-00206.safetensors b/model-00121-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..faac927923545d5d3ed14300607634f71bd9b6f6
--- /dev/null
+++ b/model-00121-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3cc6d30295ba0de3b4ed85ea1a39596ed99dd08ea01798c79d1a8184e3a67553
+size 1091293926
diff --git a/model-00122-of-00206.safetensors b/model-00122-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..e4c172245c312b379dbcd18bfd8fbd40238d4f99
--- /dev/null
+++ b/model-00122-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3b6d02db7374c3b6ed4c8f650264ebe47127a6859cef5f3eee240701b5ec8fa4
+size 5234491679
diff --git a/model-00123-of-00206.safetensors b/model-00123-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..2e151eec0e2fd0853433bc492542e473c60d625e
--- /dev/null
+++ b/model-00123-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4210c87e105ac0287002c72ac274f0a1fe206b920353af7e01690fed3cfd55ce
+size 5234491679
diff --git a/model-00124-of-00206.safetensors b/model-00124-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..e43a96ec79a5cbd6cc879e3980d40564eb0626f2
--- /dev/null
+++ b/model-00124-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:731e5e0f609429469097651e8975376f23683d0dc0a38c3b564dfc0791b1c569
+size 805306651
diff --git a/model-00125-of-00206.safetensors b/model-00125-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..c2a702763e2a69472c167e0947e8cbbd2115a041
--- /dev/null
+++ b/model-00125-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f9b851ac1442de10e9f616fbd35ce40e0b7fd80c719cbc76a671fcbc0680d60e
+size 5234491678
diff --git a/model-00126-of-00206.safetensors b/model-00126-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..820a26176792c1931afbaebc620e0ba6e3265da5
--- /dev/null
+++ b/model-00126-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c3835658d00bb15edc9a15bbf56bc212ee1e0a7d8b05100e6fb88c73c8c072de
+size 1091293960
diff --git a/model-00127-of-00206.safetensors b/model-00127-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..5be934e98ecc94b44c84970b2ddc5fac7e9b11ee
--- /dev/null
+++ b/model-00127-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:df73934e25c064e2e23c908d5e9ef0a707e7ca386fc1e11695318a7cdef2ffcf
+size 5234491679
diff --git a/model-00128-of-00206.safetensors b/model-00128-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..1ed1f2437c4c4c388fa7a8db0f46539a5dfd406b
--- /dev/null
+++ b/model-00128-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:861e3e2401af1282427eee17bc2ebb9418a1b254b68ddb64f64b44c813abf035
+size 5234491679
diff --git a/model-00129-of-00206.safetensors b/model-00129-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..b4d446a0c38f8cd833d9c588a1c4336e31fe6b0b
--- /dev/null
+++ b/model-00129-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a77ea9273bc37a987101fefe0315998403d318113a505877b2b53a0b6daaa7a5
+size 805306651
diff --git a/model-00130-of-00206.safetensors b/model-00130-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..556a13b70b5d330f67f3ab6e111cc876d3d23c24
--- /dev/null
+++ b/model-00130-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:44c74e43907460ceef7020602f4e9817a6553d0efa405b2a48cb2889db9c19d6
+size 5234491678
diff --git a/model-00131-of-00206.safetensors b/model-00131-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..fd9798ec44c90e6ec6a4b34a365098cffecd586a
--- /dev/null
+++ b/model-00131-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a20d2155fbbd52029f1eb840ff8a037bcef6a1ea7016b93d91f0afc19c574624
+size 1091293918
diff --git a/model-00132-of-00206.safetensors b/model-00132-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..78daa166fb445a5607e9411493c0b17149470aa2
--- /dev/null
+++ b/model-00132-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f9b74fd815282da59d6fbc199013eed38467ad8103742f855e93b80d8b976d59
+size 5234491679
diff --git a/model-00133-of-00206.safetensors b/model-00133-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..31eb580e676064af975f102c7032561e2ab66cd6
--- /dev/null
+++ b/model-00133-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6f8396aa5a38a2b0123e8e1bbc443163928832b365a6f8419f6bbb9fad5541f4
+size 5234491679
diff --git a/model-00134-of-00206.safetensors b/model-00134-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..b10abc8c343f49331a0a6559319934e7c7b29e03
--- /dev/null
+++ b/model-00134-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:022aa34c763a2e8d70ae75710f07709de670e8f0db39d557c5e1c558b50f8790
+size 805306651
diff --git a/model-00135-of-00206.safetensors b/model-00135-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..58aca90210735cc0ce0d3f007c3246d19a10f9d3
--- /dev/null
+++ b/model-00135-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:26531e65a33b3194f780f449f52ff69f56d83c62620e50cd595175eecf78fc25
+size 5234491678
diff --git a/model-00136-of-00206.safetensors b/model-00136-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..1d78450eaf6e9f1b815d42849000276d7e0bb393
--- /dev/null
+++ b/model-00136-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a41b2c17d4e528beb072e2d80e84c0714767f31e3875ad29d2f4a0871149ba85
+size 1091293968
diff --git a/model-00137-of-00206.safetensors b/model-00137-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..d74451bb2f5fd4317e41f43c71b3d27e30aeecac
--- /dev/null
+++ b/model-00137-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e6ba9c41bc1d5a6f2eeb3492eeed0e42182b2ab6f872c24a82c5861cfecd42d7
+size 5234491679
diff --git a/model-00138-of-00206.safetensors b/model-00138-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..e791a984f9bba2963b23e39437191864419d4e69
--- /dev/null
+++ b/model-00138-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9942d824beb5eb73b0d1d347e1d11d73d3c5e75cad228059c177e0f8b0e17a1f
+size 5234491679
diff --git a/model-00139-of-00206.safetensors b/model-00139-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..058e640a89beafa69b73da788995bef72f4b9839
--- /dev/null
+++ b/model-00139-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e9fd5ce38f18c966ff4bd6f80d1b8282129528eecd380b6ce6345e9113788da1
+size 805306651
diff --git a/model-00140-of-00206.safetensors b/model-00140-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..aa949c0ec930169110aefcf2e25ff081c7d4ff37
--- /dev/null
+++ b/model-00140-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e600647acaa06921c8f022720b581369d66830111c531e303b595f20284b9e48
+size 5234491678
diff --git a/model-00141-of-00206.safetensors b/model-00141-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..b11257a9d985589ace9a459fbfcdb925d641941e
--- /dev/null
+++ b/model-00141-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:717076ce80cc747f625dd159992c7d380cd64559329f3a4e89610d2ff1ed4082
+size 1091293964
diff --git a/model-00142-of-00206.safetensors b/model-00142-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..f805e858b003e13887aee36f7b5759d361123b2d
--- /dev/null
+++ b/model-00142-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c9f5eaffd816ac721240ecf406617004dead72b4862620f486bc3b6c6293b5fc
+size 5234491679
diff --git a/model-00143-of-00206.safetensors b/model-00143-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..0cee4742bc2daa41ecfda2a2459ae911c655ddf7
--- /dev/null
+++ b/model-00143-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b8601f370ac7250184cda7d290c7ff9a4466398ccd2921d81d351e35ecd9d0af
+size 5234491679
diff --git a/model-00144-of-00206.safetensors b/model-00144-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..88cb961e446de5da4bbb2d4bc7ee46f7b699fa31
--- /dev/null
+++ b/model-00144-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bf17873b00a4403d470a52491df8e621ef4a232f263988b613aecb6a8723ea30
+size 805306651
diff --git a/model-00145-of-00206.safetensors b/model-00145-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..14539db9b8b268ea7c6b1e9d36738afa689fb25b
--- /dev/null
+++ b/model-00145-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3f51ae407085349b27474d478af8cef548d61a6e51cc700062931ce28671203b
+size 5234491678
diff --git a/model-00146-of-00206.safetensors b/model-00146-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..2cc61657ac995198fc85a64da86889cdd0910c7b
--- /dev/null
+++ b/model-00146-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4f2e48c0e9e51d263745d65ce708490ca06edf1614b2b5906ea4b7a5593f43dd
+size 1091293940
diff --git a/model-00147-of-00206.safetensors b/model-00147-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..21222f49a5fdc57efc06df1822a49417daf56253
--- /dev/null
+++ b/model-00147-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8abbd13fbd48f6a01b182b457347d2e75dd52ccca0b0b34ef9976a58421d729d
+size 5234491679
diff --git a/model-00148-of-00206.safetensors b/model-00148-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..dd6eb7cbd00a65818ea7e36122af30065c5e9187
--- /dev/null
+++ b/model-00148-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4f9badc236a2cea396e6236d6c9452b5be5b848ed17dd09e0cbcc4ffdf39b894
+size 5234491679
diff --git a/model-00149-of-00206.safetensors b/model-00149-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..281a9c267136a638021bc25f843de5859425a0b8
--- /dev/null
+++ b/model-00149-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7aca39054f3f89ec605bb58317e50e72fa39fc1f317d58f0e6f15d9cf534ab22
+size 805306651
diff --git a/model-00150-of-00206.safetensors b/model-00150-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..c51fb4d2034e33f191f848e3b4dc28a2e27ff998
--- /dev/null
+++ b/model-00150-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d8fb6b2907eedcc56cf64f4974a0dad4ff242be73ce319d26bcfc770449e7994
+size 5234491678
diff --git a/model-00151-of-00206.safetensors b/model-00151-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..ab3cf464a04cf07490c099f251ac449cc08fd78f
--- /dev/null
+++ b/model-00151-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:998cf3badbdf26741e5ef869c84edd1ac313d0d45cc8beef1d13b1eeda8b1044
+size 1091293954
diff --git a/model-00152-of-00206.safetensors b/model-00152-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..b85e4ae52bf02c73b547e59d0da97c53fa5210ad
--- /dev/null
+++ b/model-00152-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:98c5ff967fbfebb1f2083826af860ec366e251bda6ed6b05456ae498360fe56d
+size 5234491679
diff --git a/model-00153-of-00206.safetensors b/model-00153-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..71ac4aa66dcfc6bcbdaadcdf776139c051f5c24b
--- /dev/null
+++ b/model-00153-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9ac5bde725dcc0d7287949fbc4b7984ef893ef03a40b355a922a7a41712ff3df
+size 5234491679
diff --git a/model-00154-of-00206.safetensors b/model-00154-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..e9c7062be088d5a5d371b920d4f4ab5f94ad96bf
--- /dev/null
+++ b/model-00154-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4e39cc6575ae1a8c25b96106d7e3c3657455e909a47fe79b435732199664360e
+size 805306651
diff --git a/model-00155-of-00206.safetensors b/model-00155-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..3c912df0c2988fe6cc55fa7c05c92ef0ddc7245d
--- /dev/null
+++ b/model-00155-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d9d5d4e9e2ade93ea6e6e9c4e153799d2435876d240410d548eb9ac0138f4679
+size 5234491678
diff --git a/model-00156-of-00206.safetensors b/model-00156-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..ad28fd8c2a2fbe804286164ae5f3bf1c4058bc4b
--- /dev/null
+++ b/model-00156-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a004ffd098d0fb4cabb7849d917c192f41f81b7026b8e286fdd16e32287725f7
+size 1091293948
diff --git a/model-00157-of-00206.safetensors b/model-00157-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..d346855c682e3ad086b8f301319ff8a7df510a67
--- /dev/null
+++ b/model-00157-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d420801899244667ed509e5578bdd34e9b861113ab6dfbb1fd8db044bdc4cb2e
+size 5234491679
diff --git a/model-00158-of-00206.safetensors b/model-00158-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..b29b9d5e2d28820c1281f7df5ed0fed0d45f73d1
--- /dev/null
+++ b/model-00158-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b57e2f54835530c4ec64ca58c203c5304214a8fc2ea9b2a66b10d19ad8fd5134
+size 5234491679
diff --git a/model-00159-of-00206.safetensors b/model-00159-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..767cbdc0c456ec42327526c8721b62e1fc63197d
--- /dev/null
+++ b/model-00159-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5a4e7c5f75d5485e614a660769d9f998e24a59670b1d65b2f435bdffc716ec83
+size 805306651
diff --git a/model-00160-of-00206.safetensors b/model-00160-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..1b09cd4482fa45be70bbb57de97e6874497cf24c
--- /dev/null
+++ b/model-00160-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2f0cc07218a3a88089f4993e2999246af2f54a4dbfd9867a386faf39cec62343
+size 5234491678
diff --git a/model-00161-of-00206.safetensors b/model-00161-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..4711fdd585413bcdaa1a051eac12c14c97cd58ff
--- /dev/null
+++ b/model-00161-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5264541f79990ab8ce675a6f72823e913df7d381798f884fe230f2f5bb445be9
+size 1091293942
diff --git a/model-00162-of-00206.safetensors b/model-00162-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..dc69d7e97876d922086fdb199cb54a9f60bbe251
--- /dev/null
+++ b/model-00162-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a5f09c6288c6b34f6d29b369039b3bdbb8777091975ee59ffb79d1dec075f92c
+size 5234491679
diff --git a/model-00163-of-00206.safetensors b/model-00163-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..a5ea4bde2a3d05e83417cda73582747a90210e0d
--- /dev/null
+++ b/model-00163-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7ceb5e0c144d7d5bbc4d812320eae4c79ed89ef1ea22ebc8ac68c406a34965bd
+size 5234491679
diff --git a/model-00164-of-00206.safetensors b/model-00164-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..faf06239571ca78dc5051d4d02468d9892c84564
--- /dev/null
+++ b/model-00164-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bf51fe068629e0aa22116d39dcbbd7a4c576ca3136af704f746df949ef821194
+size 805306651
diff --git a/model-00165-of-00206.safetensors b/model-00165-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..2f3bc701835048809bd4a08296836233d3b8e7a5
--- /dev/null
+++ b/model-00165-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bc229ec4e17a761d50303316a90942f188fc0cefb42d1c5b718912e9c1d4a0ab
+size 5234491678
diff --git a/model-00166-of-00206.safetensors b/model-00166-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..7b449943254c89c787505f83368a2a08d23676a5
--- /dev/null
+++ b/model-00166-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:800fb3a647ce5edb875cffaf873e2b158a9101ca8ad1cf5e43b3f1e3926af850
+size 1091293992
diff --git a/model-00167-of-00206.safetensors b/model-00167-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..6aaf771a1923208ce03d1760907d527931318504
--- /dev/null
+++ b/model-00167-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b26c67e85c62cd260a64ba6e3be9e0f9f93250c764f93af5e592e5b1f8f8a1d4
+size 5234491679
diff --git a/model-00168-of-00206.safetensors b/model-00168-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..dbfa886b73e6023fc808706f9cd226206d0aaf21
--- /dev/null
+++ b/model-00168-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f149a2e1e15598a023a91ace7b6042173a2cfb86c5c4a83a90618ea78b060301
+size 5234491679
diff --git a/model-00169-of-00206.safetensors b/model-00169-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..25e8ce1ea83ce623739ca23ec54a1606152f1d2f
--- /dev/null
+++ b/model-00169-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b9823ed6ba61959ba3c722044fa1a7a24f97a32a0f79ac64b27bc30539695738
+size 805306651
diff --git a/model-00170-of-00206.safetensors b/model-00170-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..b241e13d0babe3c7d9362d5d45031699d7ff93ff
--- /dev/null
+++ b/model-00170-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3019a4c8bff30e80d345b6bd531b785c97f16cb4ec01b9a556e475c3c4f21c8a
+size 5234491678
diff --git a/model-00171-of-00206.safetensors b/model-00171-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..8637d36ab32979af0270ccfd2da5787265c4a6b3
--- /dev/null
+++ b/model-00171-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:27b248e564804c52662b15764032b0fdf547fd70178f93a8d995201345aaf055
+size 1091293920
diff --git a/model-00172-of-00206.safetensors b/model-00172-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..0b7802c8e3fbf2fc4f879691fa24fc380cdef98a
--- /dev/null
+++ b/model-00172-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1144e1251908ea5f21d250e7ad404d883549e981aa2ba9ab5decf4db5b12f2a2
+size 5234491679
diff --git a/model-00173-of-00206.safetensors b/model-00173-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..56623b78415299b9e9a3ab7317762de7b7c44ec5
--- /dev/null
+++ b/model-00173-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5135e2aeffa143c3da3bffc32cb359203fc1524987bd28dce1287cbaf1da30cb
+size 5234491679
diff --git a/model-00174-of-00206.safetensors b/model-00174-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..f78fcd3f7dad776447fefbc3457a8df2ba1fc988
--- /dev/null
+++ b/model-00174-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d0b1e608fc66686e0f6bb860240dcc53c5f0ab9f95c48f8fe2cdc3ff6eb84cbf
+size 805306651
diff --git a/model-00175-of-00206.safetensors b/model-00175-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..5149028a382658cec9a0d8c20edde58c6a3bdf35
--- /dev/null
+++ b/model-00175-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:84159f7a1eef23c438029819ac26bef57ff731aa98d93aa0c11794cdd6effed0
+size 5234491678
diff --git a/model-00176-of-00206.safetensors b/model-00176-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..602a5fa16e80aa475f3fdb894181c0ccf75a4ba7
--- /dev/null
+++ b/model-00176-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8fa66fd411209bb2cf831b4a21b10060a333c4ad32ba820983a96f1480c565c7
+size 1091293924
diff --git a/model-00177-of-00206.safetensors b/model-00177-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..8db2fefbac78a6bc62a36ef3035efa8d1308dc1d
--- /dev/null
+++ b/model-00177-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:76ff0fe39bf9efd079e6860cc8d1f32f9ff098ad6446aedcce474c77f571db35
+size 5234491679
diff --git a/model-00178-of-00206.safetensors b/model-00178-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..d6648c3e3b29e1d9b313c07e580e0d781dced81a
--- /dev/null
+++ b/model-00178-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:17845b49e6d0fcdc249c363c02b65f68b2186efa643cb7d53db47a3a574140d9
+size 5234491679
diff --git a/model-00179-of-00206.safetensors b/model-00179-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..5ceb662fa17144fb51dfe65ae6712f6415a088aa
--- /dev/null
+++ b/model-00179-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3f6feff3f9922cfe5706c0985f47cd9a7d0189c4e7d81d7b9310175a8608b0d1
+size 805306651
diff --git a/model-00180-of-00206.safetensors b/model-00180-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..81164bd6d4e07f9d7b6e60e55d0518f9b113536a
--- /dev/null
+++ b/model-00180-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:59e807d00c146a147f315b4c906e370a2dce744cfdd0dc33fafd6c802a548382
+size 5234491678
diff --git a/model-00181-of-00206.safetensors b/model-00181-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..d1fa266b23f56ad8eee741720c098c1fcd4e5449
--- /dev/null
+++ b/model-00181-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1bc36980779c4d8618a00b2090c78b442be070a7dc4c4756e3e32f2a2eeda6f1
+size 1091293916
diff --git a/model-00182-of-00206.safetensors b/model-00182-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..c2fd55e7beb0de66098c24abf00bea97b13e3a2a
--- /dev/null
+++ b/model-00182-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3a3e81e805e7e67a05f531427cfa5ea5691653addd38d1e82f9033a46eb0d779
+size 5234491679
diff --git a/model-00183-of-00206.safetensors b/model-00183-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..9ac0dbff845ddd70a0501043f668ba9310cf4b09
--- /dev/null
+++ b/model-00183-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4563c5255551d1b000c57cf286fff92b8576e2ff718176d73a1d9c22b1a4e171
+size 5234491679
diff --git a/model-00184-of-00206.safetensors b/model-00184-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..0755ae62371b0cc622c63b23d0b86411f52abd39
--- /dev/null
+++ b/model-00184-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a4894e0df2b0ec32f8d3b26bf45d2a1b0d92ecb5462d843739a1ba0505a5d7b9
+size 805306651
diff --git a/model-00185-of-00206.safetensors b/model-00185-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..9c233f2f00c39c8c88846474aa794122e0a9da99
--- /dev/null
+++ b/model-00185-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b8ee811a38153d589d38fbbd448ab1599da83d9848849520d8250731508a5726
+size 5234491678
diff --git a/model-00186-of-00206.safetensors b/model-00186-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..8b017515e8bd70352466f52da2a6ac93fb785323
--- /dev/null
+++ b/model-00186-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4f9cb49e0cb375afe597e21b328b125603f02b3a161a34a5526538511867f578
+size 1091293946
diff --git a/model-00187-of-00206.safetensors b/model-00187-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..568f40c99cc94c5e5cb5803f11963e2f7f9f6d9e
--- /dev/null
+++ b/model-00187-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5d4553c91af529602b079b66deb304e3d81ebd82bdaffbeb4e84a42b17af25cd
+size 5234491679
diff --git a/model-00188-of-00206.safetensors b/model-00188-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..2a0e13b237fc8c8b2128aa7559f86502d0e72e50
--- /dev/null
+++ b/model-00188-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:64167d10edbaa3aa48032177a560adeb8df2400907fd8d5b53381ce3c35d9647
+size 5234491679
diff --git a/model-00189-of-00206.safetensors b/model-00189-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..058357ebd2de1f40e5a9d111beaef1e97bd190cb
--- /dev/null
+++ b/model-00189-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d8008df08ff1f1de95ee3b2e6574ab893378044fdfc8c13e7f97c719b8175780
+size 805306651
diff --git a/model-00190-of-00206.safetensors b/model-00190-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..0bacd4d54876eccdc5b67b54b0a208e43c87a338
--- /dev/null
+++ b/model-00190-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b7d5f7c912ce137a12ef855dd2f6cf44e4f9bc1f3cb1dfbbd84ba231fda7c7e8
+size 5234491678
diff --git a/model-00191-of-00206.safetensors b/model-00191-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..9ddae498d4313e713b476423d1c69d10e24c3513
--- /dev/null
+++ b/model-00191-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3ad8a2e012e51cd6a84d96e9462843ec948b5eff4a7d282a7f28880306e58c7d
+size 1091293954
diff --git a/model-00192-of-00206.safetensors b/model-00192-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..8737d8fd799b5e4ae58302c0525d8ce69bda0bdb
--- /dev/null
+++ b/model-00192-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:71feefa1239e5a091a62b51154be09cdf52b82ce79e8e92831f8d30f711c7b60
+size 5234491679
diff --git a/model-00193-of-00206.safetensors b/model-00193-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..aec0632db89e4351ef754ed267d4c59e8d983d9f
--- /dev/null
+++ b/model-00193-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a62e4cb7805ea8ecd8be84f3d0de0b84bd1f4919f2ebc79c51584f8458e7b53c
+size 5234491679
diff --git a/model-00194-of-00206.safetensors b/model-00194-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..5d2fc437744077c3ee2ed197c5cda4acb0119879
--- /dev/null
+++ b/model-00194-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3477a23ff2670c2ac00f6932b9764425e4b6d27ae8b33228f940ca606be87899
+size 805306651
diff --git a/model-00195-of-00206.safetensors b/model-00195-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..a8e6bb400dabff05be58a2d70c1ed4fb0ff83eae
--- /dev/null
+++ b/model-00195-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:859fd6fdb31991aeb8ad2e6383521e09ed69224dd7592f6226bfcbbbd2313be3
+size 5234491678
diff --git a/model-00196-of-00206.safetensors b/model-00196-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..e400233646d47da12351ab9641e52d4e17850374
--- /dev/null
+++ b/model-00196-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3e779be7a4b53f9d07341e32273dcb9ebd1756bdbbdfa51839006f8ec8ab2a3d
+size 1091293932
diff --git a/model-00197-of-00206.safetensors b/model-00197-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..1a07c63062104f64adf7b9f294f01052d83e6dcb
--- /dev/null
+++ b/model-00197-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b379861780af1491f2410bf0dc76d81e32453c4fe3afb58ef60a9a69a1f58633
+size 5234491679
diff --git a/model-00198-of-00206.safetensors b/model-00198-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..0ab6ec47ed1e9b9b9bd419ec9185cce7ab2c79df
--- /dev/null
+++ b/model-00198-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:da34ef554aef9019949de4fa35dcaba38d89d47643367dd9dfd8a9ddc8230468
+size 5234491679
diff --git a/model-00199-of-00206.safetensors b/model-00199-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..ddaaa1d1a829391e3e58ba703e6d9a490501c2cd
--- /dev/null
+++ b/model-00199-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9020029ea531194a4299397de9557943cb1723fed035707ef4ebcf3e2631710f
+size 805306651
diff --git a/model-00200-of-00206.safetensors b/model-00200-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..26c377a20c78b2e99ea8fe2ade1f3f0e02781b25
--- /dev/null
+++ b/model-00200-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a6a1f4c06b418060ee985a2c8ea516f0bb601d244ab0c1aa2e31e151f45bca49
+size 5234491678
diff --git a/model-00201-of-00206.safetensors b/model-00201-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..6f738416da986fd1fae61c26f93d5178f21dee0e
--- /dev/null
+++ b/model-00201-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:79aae0cc29a3a025579df8783152499958d3420e767c449f39e8c1251594431d
+size 1091293940
diff --git a/model-00202-of-00206.safetensors b/model-00202-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..e3eb24ba7b082db0aebe374c81defc61e5d94fd7
--- /dev/null
+++ b/model-00202-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ca30dff5cae6e5796cc7fc330ff8c07841603f9e784a6e072a66b8662b71fbc9
+size 5234491679
diff --git a/model-00203-of-00206.safetensors b/model-00203-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..79ab84d3585cac97a8b41679819059550047d919
--- /dev/null
+++ b/model-00203-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0ba766d6d2b7a9711c4c9f840bd200d854688fa6e8a0aa221f72cbf72ce3cdc9
+size 5234491679
diff --git a/model-00204-of-00206.safetensors b/model-00204-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..352d691b5a6fc165c857bbab1c66e990beb858f5
--- /dev/null
+++ b/model-00204-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:47bcb41deb0ac4e76047a36e2de24421dc4d99e0e1087f8337f6da9f734aae44
+size 805306651
diff --git a/model-00205-of-00206.safetensors b/model-00205-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..5497e873f60ab4c755dceb9956e774950c29b6f4
--- /dev/null
+++ b/model-00205-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:601f2814fa7a5127ff921c448f60626e882fd646cad964ed45d6b2b7e93b8417
+size 5234491678
diff --git a/model-00206-of-00206.safetensors b/model-00206-of-00206.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..8a665b09e0efb02f523f418a41fb17859ef43647
--- /dev/null
+++ b/model-00206-of-00206.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:78f9136eea4c295c346fc418f592ef6d061247a1e8f99747e04a2f8a7838f780
+size 2181829783
diff --git a/model.safetensors.index.json b/model.safetensors.index.json
new file mode 100644
index 0000000000000000000000000000000000000000..4e18c40f8120bb8dbb8ff27a39781988e4eab38c
--- /dev/null
+++ b/model.safetensors.index.json
@@ -0,0 +1,3341 @@
+{
+ "metadata": {
+ "total_size": 730091317248,
+ "total_parameters": 1638616773376
+ },
+ "weight_map": {
+ "lm_head.biases": "model-00206-of-00206.safetensors",
+ "lm_head.scales": "model-00206-of-00206.safetensors",
+ "lm_head.weight": "model-00206-of-00206.safetensors",
+ "model.layers.0.input_layernorm.0.weight": "model-00021-of-00206.safetensors",
+ "model.layers.0.input_layernorm.1.weight": "model-00021-of-00206.safetensors",
+ "model.layers.0.mlp.router.classifier.biases": "model-00021-of-00206.safetensors",
+ "model.layers.0.mlp.router.classifier.scales": "model-00021-of-00206.safetensors",
+ "model.layers.0.mlp.router.classifier.weight": "model-00021-of-00206.safetensors",
+ "model.layers.0.mlp.router.e_score_correction_bias": "model-00021-of-00206.safetensors",
+ "model.layers.0.mlp.switch_mlp.down_proj.biases": "model-00021-of-00206.safetensors",
+ "model.layers.0.mlp.switch_mlp.down_proj.scales": "model-00020-of-00206.safetensors",
+ "model.layers.0.mlp.switch_mlp.down_proj.weight": "model-00020-of-00206.safetensors",
+ "model.layers.0.mlp.switch_mlp.gate_proj.biases": "model-00018-of-00206.safetensors",
+ "model.layers.0.mlp.switch_mlp.gate_proj.scales": "model-00017-of-00206.safetensors",
+ "model.layers.0.mlp.switch_mlp.gate_proj.weight": "model-00017-of-00206.safetensors",
+ "model.layers.0.mlp.switch_mlp.up_proj.biases": "model-00019-of-00206.safetensors",
+ "model.layers.0.mlp.switch_mlp.up_proj.scales": "model-00019-of-00206.safetensors",
+ "model.layers.0.mlp.switch_mlp.up_proj.weight": "model-00018-of-00206.safetensors",
+ "model.layers.0.mlps.0.down_proj.biases": "model-00021-of-00206.safetensors",
+ "model.layers.0.mlps.0.down_proj.scales": "model-00021-of-00206.safetensors",
+ "model.layers.0.mlps.0.down_proj.weight": "model-00021-of-00206.safetensors",
+ "model.layers.0.mlps.0.gate_proj.biases": "model-00021-of-00206.safetensors",
+ "model.layers.0.mlps.0.gate_proj.scales": "model-00021-of-00206.safetensors",
+ "model.layers.0.mlps.0.gate_proj.weight": "model-00021-of-00206.safetensors",
+ "model.layers.0.mlps.0.up_proj.biases": "model-00021-of-00206.safetensors",
+ "model.layers.0.mlps.0.up_proj.scales": "model-00021-of-00206.safetensors",
+ "model.layers.0.mlps.0.up_proj.weight": "model-00021-of-00206.safetensors",
+ "model.layers.0.mlps.1.down_proj.biases": "model-00021-of-00206.safetensors",
+ "model.layers.0.mlps.1.down_proj.scales": "model-00021-of-00206.safetensors",
+ "model.layers.0.mlps.1.down_proj.weight": "model-00021-of-00206.safetensors",
+ "model.layers.0.mlps.1.gate_proj.biases": "model-00021-of-00206.safetensors",
+ "model.layers.0.mlps.1.gate_proj.scales": "model-00021-of-00206.safetensors",
+ "model.layers.0.mlps.1.gate_proj.weight": "model-00021-of-00206.safetensors",
+ "model.layers.0.mlps.1.up_proj.biases": "model-00021-of-00206.safetensors",
+ "model.layers.0.mlps.1.up_proj.scales": "model-00021-of-00206.safetensors",
+ "model.layers.0.mlps.1.up_proj.weight": "model-00021-of-00206.safetensors",
+ "model.layers.0.post_attention_layernorm.0.weight": "model-00021-of-00206.safetensors",
+ "model.layers.0.post_attention_layernorm.1.weight": "model-00021-of-00206.safetensors",
+ "model.layers.0.self_attn.0.embed_q.biases": "model-00021-of-00206.safetensors",
+ "model.layers.0.self_attn.0.embed_q.scales": "model-00021-of-00206.safetensors",
+ "model.layers.0.self_attn.0.embed_q.weight": "model-00021-of-00206.safetensors",
+ "model.layers.0.self_attn.0.indexer.k_norm.weight": "model-00021-of-00206.safetensors",
+ "model.layers.0.self_attn.0.indexer.weights_proj.biases": "model-00021-of-00206.safetensors",
+ "model.layers.0.self_attn.0.indexer.weights_proj.scales": "model-00021-of-00206.safetensors",
+ "model.layers.0.self_attn.0.indexer.weights_proj.weight": "model-00021-of-00206.safetensors",
+ "model.layers.0.self_attn.0.indexer.wk.biases": "model-00021-of-00206.safetensors",
+ "model.layers.0.self_attn.0.indexer.wk.scales": "model-00021-of-00206.safetensors",
+ "model.layers.0.self_attn.0.indexer.wk.weight": "model-00021-of-00206.safetensors",
+ "model.layers.0.self_attn.0.indexer.wq_b.biases": "model-00021-of-00206.safetensors",
+ "model.layers.0.self_attn.0.indexer.wq_b.scales": "model-00021-of-00206.safetensors",
+ "model.layers.0.self_attn.0.indexer.wq_b.weight": "model-00021-of-00206.safetensors",
+ "model.layers.0.self_attn.0.kv_a_layernorm.weight": "model-00021-of-00206.safetensors",
+ "model.layers.0.self_attn.0.kv_a_proj_with_mqa.biases": "model-00021-of-00206.safetensors",
+ "model.layers.0.self_attn.0.kv_a_proj_with_mqa.scales": "model-00021-of-00206.safetensors",
+ "model.layers.0.self_attn.0.kv_a_proj_with_mqa.weight": "model-00021-of-00206.safetensors",
+ "model.layers.0.self_attn.0.o_proj.biases": "model-00021-of-00206.safetensors",
+ "model.layers.0.self_attn.0.o_proj.scales": "model-00021-of-00206.safetensors",
+ "model.layers.0.self_attn.0.o_proj.weight": "model-00021-of-00206.safetensors",
+ "model.layers.0.self_attn.0.q_a_layernorm.weight": "model-00021-of-00206.safetensors",
+ "model.layers.0.self_attn.0.q_a_proj.biases": "model-00021-of-00206.safetensors",
+ "model.layers.0.self_attn.0.q_a_proj.scales": "model-00021-of-00206.safetensors",
+ "model.layers.0.self_attn.0.q_a_proj.weight": "model-00021-of-00206.safetensors",
+ "model.layers.0.self_attn.0.q_b_proj.biases": "model-00021-of-00206.safetensors",
+ "model.layers.0.self_attn.0.q_b_proj.scales": "model-00021-of-00206.safetensors",
+ "model.layers.0.self_attn.0.q_b_proj.weight": "model-00021-of-00206.safetensors",
+ "model.layers.0.self_attn.0.unembed_out.biases": "model-00021-of-00206.safetensors",
+ "model.layers.0.self_attn.0.unembed_out.scales": "model-00021-of-00206.safetensors",
+ "model.layers.0.self_attn.0.unembed_out.weight": "model-00021-of-00206.safetensors",
+ "model.layers.0.self_attn.1.embed_q.biases": "model-00021-of-00206.safetensors",
+ "model.layers.0.self_attn.1.embed_q.scales": "model-00021-of-00206.safetensors",
+ "model.layers.0.self_attn.1.embed_q.weight": "model-00021-of-00206.safetensors",
+ "model.layers.0.self_attn.1.kv_a_layernorm.weight": "model-00021-of-00206.safetensors",
+ "model.layers.0.self_attn.1.kv_a_proj_with_mqa.biases": "model-00021-of-00206.safetensors",
+ "model.layers.0.self_attn.1.kv_a_proj_with_mqa.scales": "model-00021-of-00206.safetensors",
+ "model.layers.0.self_attn.1.kv_a_proj_with_mqa.weight": "model-00021-of-00206.safetensors",
+ "model.layers.0.self_attn.1.o_proj.biases": "model-00021-of-00206.safetensors",
+ "model.layers.0.self_attn.1.o_proj.scales": "model-00021-of-00206.safetensors",
+ "model.layers.0.self_attn.1.o_proj.weight": "model-00021-of-00206.safetensors",
+ "model.layers.0.self_attn.1.q_a_layernorm.weight": "model-00021-of-00206.safetensors",
+ "model.layers.0.self_attn.1.q_a_proj.biases": "model-00021-of-00206.safetensors",
+ "model.layers.0.self_attn.1.q_a_proj.scales": "model-00021-of-00206.safetensors",
+ "model.layers.0.self_attn.1.q_a_proj.weight": "model-00021-of-00206.safetensors",
+ "model.layers.0.self_attn.1.q_b_proj.biases": "model-00021-of-00206.safetensors",
+ "model.layers.0.self_attn.1.q_b_proj.scales": "model-00021-of-00206.safetensors",
+ "model.layers.0.self_attn.1.q_b_proj.weight": "model-00021-of-00206.safetensors",
+ "model.layers.0.self_attn.1.unembed_out.biases": "model-00021-of-00206.safetensors",
+ "model.layers.0.self_attn.1.unembed_out.scales": "model-00021-of-00206.safetensors",
+ "model.layers.0.self_attn.1.unembed_out.weight": "model-00021-of-00206.safetensors",
+ "model.layers.1.input_layernorm.0.weight": "model-00026-of-00206.safetensors",
+ "model.layers.1.input_layernorm.1.weight": "model-00026-of-00206.safetensors",
+ "model.layers.1.mlp.router.classifier.biases": "model-00026-of-00206.safetensors",
+ "model.layers.1.mlp.router.classifier.scales": "model-00026-of-00206.safetensors",
+ "model.layers.1.mlp.router.classifier.weight": "model-00026-of-00206.safetensors",
+ "model.layers.1.mlp.router.e_score_correction_bias": "model-00026-of-00206.safetensors",
+ "model.layers.1.mlp.switch_mlp.down_proj.biases": "model-00026-of-00206.safetensors",
+ "model.layers.1.mlp.switch_mlp.down_proj.scales": "model-00025-of-00206.safetensors",
+ "model.layers.1.mlp.switch_mlp.down_proj.weight": "model-00025-of-00206.safetensors",
+ "model.layers.1.mlp.switch_mlp.gate_proj.biases": "model-00023-of-00206.safetensors",
+ "model.layers.1.mlp.switch_mlp.gate_proj.scales": "model-00022-of-00206.safetensors",
+ "model.layers.1.mlp.switch_mlp.gate_proj.weight": "model-00022-of-00206.safetensors",
+ "model.layers.1.mlp.switch_mlp.up_proj.biases": "model-00024-of-00206.safetensors",
+ "model.layers.1.mlp.switch_mlp.up_proj.scales": "model-00024-of-00206.safetensors",
+ "model.layers.1.mlp.switch_mlp.up_proj.weight": "model-00023-of-00206.safetensors",
+ "model.layers.1.mlps.0.down_proj.biases": "model-00026-of-00206.safetensors",
+ "model.layers.1.mlps.0.down_proj.scales": "model-00026-of-00206.safetensors",
+ "model.layers.1.mlps.0.down_proj.weight": "model-00026-of-00206.safetensors",
+ "model.layers.1.mlps.0.gate_proj.biases": "model-00026-of-00206.safetensors",
+ "model.layers.1.mlps.0.gate_proj.scales": "model-00026-of-00206.safetensors",
+ "model.layers.1.mlps.0.gate_proj.weight": "model-00026-of-00206.safetensors",
+ "model.layers.1.mlps.0.up_proj.biases": "model-00026-of-00206.safetensors",
+ "model.layers.1.mlps.0.up_proj.scales": "model-00026-of-00206.safetensors",
+ "model.layers.1.mlps.0.up_proj.weight": "model-00026-of-00206.safetensors",
+ "model.layers.1.mlps.1.down_proj.biases": "model-00026-of-00206.safetensors",
+ "model.layers.1.mlps.1.down_proj.scales": "model-00026-of-00206.safetensors",
+ "model.layers.1.mlps.1.down_proj.weight": "model-00026-of-00206.safetensors",
+ "model.layers.1.mlps.1.gate_proj.biases": "model-00026-of-00206.safetensors",
+ "model.layers.1.mlps.1.gate_proj.scales": "model-00026-of-00206.safetensors",
+ "model.layers.1.mlps.1.gate_proj.weight": "model-00026-of-00206.safetensors",
+ "model.layers.1.mlps.1.up_proj.biases": "model-00026-of-00206.safetensors",
+ "model.layers.1.mlps.1.up_proj.scales": "model-00026-of-00206.safetensors",
+ "model.layers.1.mlps.1.up_proj.weight": "model-00026-of-00206.safetensors",
+ "model.layers.1.post_attention_layernorm.0.weight": "model-00026-of-00206.safetensors",
+ "model.layers.1.post_attention_layernorm.1.weight": "model-00026-of-00206.safetensors",
+ "model.layers.1.self_attn.0.embed_q.biases": "model-00026-of-00206.safetensors",
+ "model.layers.1.self_attn.0.embed_q.scales": "model-00026-of-00206.safetensors",
+ "model.layers.1.self_attn.0.embed_q.weight": "model-00026-of-00206.safetensors",
+ "model.layers.1.self_attn.0.indexer.k_norm.weight": "model-00026-of-00206.safetensors",
+ "model.layers.1.self_attn.0.indexer.weights_proj.biases": "model-00026-of-00206.safetensors",
+ "model.layers.1.self_attn.0.indexer.weights_proj.scales": "model-00026-of-00206.safetensors",
+ "model.layers.1.self_attn.0.indexer.weights_proj.weight": "model-00026-of-00206.safetensors",
+ "model.layers.1.self_attn.0.indexer.wk.biases": "model-00026-of-00206.safetensors",
+ "model.layers.1.self_attn.0.indexer.wk.scales": "model-00026-of-00206.safetensors",
+ "model.layers.1.self_attn.0.indexer.wk.weight": "model-00026-of-00206.safetensors",
+ "model.layers.1.self_attn.0.indexer.wq_b.biases": "model-00026-of-00206.safetensors",
+ "model.layers.1.self_attn.0.indexer.wq_b.scales": "model-00026-of-00206.safetensors",
+ "model.layers.1.self_attn.0.indexer.wq_b.weight": "model-00026-of-00206.safetensors",
+ "model.layers.1.self_attn.0.kv_a_layernorm.weight": "model-00026-of-00206.safetensors",
+ "model.layers.1.self_attn.0.kv_a_proj_with_mqa.biases": "model-00026-of-00206.safetensors",
+ "model.layers.1.self_attn.0.kv_a_proj_with_mqa.scales": "model-00026-of-00206.safetensors",
+ "model.layers.1.self_attn.0.kv_a_proj_with_mqa.weight": "model-00026-of-00206.safetensors",
+ "model.layers.1.self_attn.0.o_proj.biases": "model-00026-of-00206.safetensors",
+ "model.layers.1.self_attn.0.o_proj.scales": "model-00026-of-00206.safetensors",
+ "model.layers.1.self_attn.0.o_proj.weight": "model-00026-of-00206.safetensors",
+ "model.layers.1.self_attn.0.q_a_layernorm.weight": "model-00026-of-00206.safetensors",
+ "model.layers.1.self_attn.0.q_a_proj.biases": "model-00026-of-00206.safetensors",
+ "model.layers.1.self_attn.0.q_a_proj.scales": "model-00026-of-00206.safetensors",
+ "model.layers.1.self_attn.0.q_a_proj.weight": "model-00026-of-00206.safetensors",
+ "model.layers.1.self_attn.0.q_b_proj.biases": "model-00026-of-00206.safetensors",
+ "model.layers.1.self_attn.0.q_b_proj.scales": "model-00026-of-00206.safetensors",
+ "model.layers.1.self_attn.0.q_b_proj.weight": "model-00026-of-00206.safetensors",
+ "model.layers.1.self_attn.0.unembed_out.biases": "model-00026-of-00206.safetensors",
+ "model.layers.1.self_attn.0.unembed_out.scales": "model-00026-of-00206.safetensors",
+ "model.layers.1.self_attn.0.unembed_out.weight": "model-00026-of-00206.safetensors",
+ "model.layers.1.self_attn.1.embed_q.biases": "model-00026-of-00206.safetensors",
+ "model.layers.1.self_attn.1.embed_q.scales": "model-00026-of-00206.safetensors",
+ "model.layers.1.self_attn.1.embed_q.weight": "model-00026-of-00206.safetensors",
+ "model.layers.1.self_attn.1.kv_a_layernorm.weight": "model-00026-of-00206.safetensors",
+ "model.layers.1.self_attn.1.kv_a_proj_with_mqa.biases": "model-00026-of-00206.safetensors",
+ "model.layers.1.self_attn.1.kv_a_proj_with_mqa.scales": "model-00026-of-00206.safetensors",
+ "model.layers.1.self_attn.1.kv_a_proj_with_mqa.weight": "model-00026-of-00206.safetensors",
+ "model.layers.1.self_attn.1.o_proj.biases": "model-00026-of-00206.safetensors",
+ "model.layers.1.self_attn.1.o_proj.scales": "model-00026-of-00206.safetensors",
+ "model.layers.1.self_attn.1.o_proj.weight": "model-00026-of-00206.safetensors",
+ "model.layers.1.self_attn.1.q_a_layernorm.weight": "model-00026-of-00206.safetensors",
+ "model.layers.1.self_attn.1.q_a_proj.biases": "model-00026-of-00206.safetensors",
+ "model.layers.1.self_attn.1.q_a_proj.scales": "model-00026-of-00206.safetensors",
+ "model.layers.1.self_attn.1.q_a_proj.weight": "model-00026-of-00206.safetensors",
+ "model.layers.1.self_attn.1.q_b_proj.biases": "model-00026-of-00206.safetensors",
+ "model.layers.1.self_attn.1.q_b_proj.scales": "model-00026-of-00206.safetensors",
+ "model.layers.1.self_attn.1.q_b_proj.weight": "model-00026-of-00206.safetensors",
+ "model.layers.1.self_attn.1.unembed_out.biases": "model-00026-of-00206.safetensors",
+ "model.layers.1.self_attn.1.unembed_out.scales": "model-00026-of-00206.safetensors",
+ "model.layers.1.self_attn.1.unembed_out.weight": "model-00026-of-00206.safetensors",
+ "model.layers.10.input_layernorm.0.weight": "model-00071-of-00206.safetensors",
+ "model.layers.10.input_layernorm.1.weight": "model-00071-of-00206.safetensors",
+ "model.layers.10.mlp.router.classifier.biases": "model-00071-of-00206.safetensors",
+ "model.layers.10.mlp.router.classifier.scales": "model-00071-of-00206.safetensors",
+ "model.layers.10.mlp.router.classifier.weight": "model-00071-of-00206.safetensors",
+ "model.layers.10.mlp.router.e_score_correction_bias": "model-00071-of-00206.safetensors",
+ "model.layers.10.mlp.switch_mlp.down_proj.biases": "model-00071-of-00206.safetensors",
+ "model.layers.10.mlp.switch_mlp.down_proj.scales": "model-00070-of-00206.safetensors",
+ "model.layers.10.mlp.switch_mlp.down_proj.weight": "model-00070-of-00206.safetensors",
+ "model.layers.10.mlp.switch_mlp.gate_proj.biases": "model-00068-of-00206.safetensors",
+ "model.layers.10.mlp.switch_mlp.gate_proj.scales": "model-00067-of-00206.safetensors",
+ "model.layers.10.mlp.switch_mlp.gate_proj.weight": "model-00067-of-00206.safetensors",
+ "model.layers.10.mlp.switch_mlp.up_proj.biases": "model-00069-of-00206.safetensors",
+ "model.layers.10.mlp.switch_mlp.up_proj.scales": "model-00069-of-00206.safetensors",
+ "model.layers.10.mlp.switch_mlp.up_proj.weight": "model-00068-of-00206.safetensors",
+ "model.layers.10.mlps.0.down_proj.biases": "model-00071-of-00206.safetensors",
+ "model.layers.10.mlps.0.down_proj.scales": "model-00071-of-00206.safetensors",
+ "model.layers.10.mlps.0.down_proj.weight": "model-00071-of-00206.safetensors",
+ "model.layers.10.mlps.0.gate_proj.biases": "model-00071-of-00206.safetensors",
+ "model.layers.10.mlps.0.gate_proj.scales": "model-00071-of-00206.safetensors",
+ "model.layers.10.mlps.0.gate_proj.weight": "model-00071-of-00206.safetensors",
+ "model.layers.10.mlps.0.up_proj.biases": "model-00071-of-00206.safetensors",
+ "model.layers.10.mlps.0.up_proj.scales": "model-00071-of-00206.safetensors",
+ "model.layers.10.mlps.0.up_proj.weight": "model-00071-of-00206.safetensors",
+ "model.layers.10.mlps.1.down_proj.biases": "model-00071-of-00206.safetensors",
+ "model.layers.10.mlps.1.down_proj.scales": "model-00071-of-00206.safetensors",
+ "model.layers.10.mlps.1.down_proj.weight": "model-00071-of-00206.safetensors",
+ "model.layers.10.mlps.1.gate_proj.biases": "model-00071-of-00206.safetensors",
+ "model.layers.10.mlps.1.gate_proj.scales": "model-00071-of-00206.safetensors",
+ "model.layers.10.mlps.1.gate_proj.weight": "model-00071-of-00206.safetensors",
+ "model.layers.10.mlps.1.up_proj.biases": "model-00071-of-00206.safetensors",
+ "model.layers.10.mlps.1.up_proj.scales": "model-00071-of-00206.safetensors",
+ "model.layers.10.mlps.1.up_proj.weight": "model-00071-of-00206.safetensors",
+ "model.layers.10.post_attention_layernorm.0.weight": "model-00071-of-00206.safetensors",
+ "model.layers.10.post_attention_layernorm.1.weight": "model-00071-of-00206.safetensors",
+ "model.layers.10.self_attn.0.embed_q.biases": "model-00071-of-00206.safetensors",
+ "model.layers.10.self_attn.0.embed_q.scales": "model-00071-of-00206.safetensors",
+ "model.layers.10.self_attn.0.embed_q.weight": "model-00071-of-00206.safetensors",
+ "model.layers.10.self_attn.0.indexer.k_norm.weight": "model-00071-of-00206.safetensors",
+ "model.layers.10.self_attn.0.indexer.weights_proj.biases": "model-00071-of-00206.safetensors",
+ "model.layers.10.self_attn.0.indexer.weights_proj.scales": "model-00071-of-00206.safetensors",
+ "model.layers.10.self_attn.0.indexer.weights_proj.weight": "model-00071-of-00206.safetensors",
+ "model.layers.10.self_attn.0.indexer.wk.biases": "model-00071-of-00206.safetensors",
+ "model.layers.10.self_attn.0.indexer.wk.scales": "model-00071-of-00206.safetensors",
+ "model.layers.10.self_attn.0.indexer.wk.weight": "model-00071-of-00206.safetensors",
+ "model.layers.10.self_attn.0.indexer.wq_b.biases": "model-00071-of-00206.safetensors",
+ "model.layers.10.self_attn.0.indexer.wq_b.scales": "model-00071-of-00206.safetensors",
+ "model.layers.10.self_attn.0.indexer.wq_b.weight": "model-00071-of-00206.safetensors",
+ "model.layers.10.self_attn.0.kv_a_layernorm.weight": "model-00071-of-00206.safetensors",
+ "model.layers.10.self_attn.0.kv_a_proj_with_mqa.biases": "model-00071-of-00206.safetensors",
+ "model.layers.10.self_attn.0.kv_a_proj_with_mqa.scales": "model-00071-of-00206.safetensors",
+ "model.layers.10.self_attn.0.kv_a_proj_with_mqa.weight": "model-00071-of-00206.safetensors",
+ "model.layers.10.self_attn.0.o_proj.biases": "model-00071-of-00206.safetensors",
+ "model.layers.10.self_attn.0.o_proj.scales": "model-00071-of-00206.safetensors",
+ "model.layers.10.self_attn.0.o_proj.weight": "model-00071-of-00206.safetensors",
+ "model.layers.10.self_attn.0.q_a_layernorm.weight": "model-00071-of-00206.safetensors",
+ "model.layers.10.self_attn.0.q_a_proj.biases": "model-00071-of-00206.safetensors",
+ "model.layers.10.self_attn.0.q_a_proj.scales": "model-00071-of-00206.safetensors",
+ "model.layers.10.self_attn.0.q_a_proj.weight": "model-00071-of-00206.safetensors",
+ "model.layers.10.self_attn.0.q_b_proj.biases": "model-00071-of-00206.safetensors",
+ "model.layers.10.self_attn.0.q_b_proj.scales": "model-00071-of-00206.safetensors",
+ "model.layers.10.self_attn.0.q_b_proj.weight": "model-00071-of-00206.safetensors",
+ "model.layers.10.self_attn.0.unembed_out.biases": "model-00071-of-00206.safetensors",
+ "model.layers.10.self_attn.0.unembed_out.scales": "model-00071-of-00206.safetensors",
+ "model.layers.10.self_attn.0.unembed_out.weight": "model-00071-of-00206.safetensors",
+ "model.layers.10.self_attn.1.embed_q.biases": "model-00071-of-00206.safetensors",
+ "model.layers.10.self_attn.1.embed_q.scales": "model-00071-of-00206.safetensors",
+ "model.layers.10.self_attn.1.embed_q.weight": "model-00071-of-00206.safetensors",
+ "model.layers.10.self_attn.1.kv_a_layernorm.weight": "model-00071-of-00206.safetensors",
+ "model.layers.10.self_attn.1.kv_a_proj_with_mqa.biases": "model-00071-of-00206.safetensors",
+ "model.layers.10.self_attn.1.kv_a_proj_with_mqa.scales": "model-00071-of-00206.safetensors",
+ "model.layers.10.self_attn.1.kv_a_proj_with_mqa.weight": "model-00071-of-00206.safetensors",
+ "model.layers.10.self_attn.1.o_proj.biases": "model-00071-of-00206.safetensors",
+ "model.layers.10.self_attn.1.o_proj.scales": "model-00071-of-00206.safetensors",
+ "model.layers.10.self_attn.1.o_proj.weight": "model-00071-of-00206.safetensors",
+ "model.layers.10.self_attn.1.q_a_layernorm.weight": "model-00071-of-00206.safetensors",
+ "model.layers.10.self_attn.1.q_a_proj.biases": "model-00071-of-00206.safetensors",
+ "model.layers.10.self_attn.1.q_a_proj.scales": "model-00071-of-00206.safetensors",
+ "model.layers.10.self_attn.1.q_a_proj.weight": "model-00071-of-00206.safetensors",
+ "model.layers.10.self_attn.1.q_b_proj.biases": "model-00071-of-00206.safetensors",
+ "model.layers.10.self_attn.1.q_b_proj.scales": "model-00071-of-00206.safetensors",
+ "model.layers.10.self_attn.1.q_b_proj.weight": "model-00071-of-00206.safetensors",
+ "model.layers.10.self_attn.1.unembed_out.biases": "model-00071-of-00206.safetensors",
+ "model.layers.10.self_attn.1.unembed_out.scales": "model-00071-of-00206.safetensors",
+ "model.layers.10.self_attn.1.unembed_out.weight": "model-00071-of-00206.safetensors",
+ "model.layers.11.input_layernorm.0.weight": "model-00076-of-00206.safetensors",
+ "model.layers.11.input_layernorm.1.weight": "model-00076-of-00206.safetensors",
+ "model.layers.11.mlp.router.classifier.biases": "model-00076-of-00206.safetensors",
+ "model.layers.11.mlp.router.classifier.scales": "model-00076-of-00206.safetensors",
+ "model.layers.11.mlp.router.classifier.weight": "model-00076-of-00206.safetensors",
+ "model.layers.11.mlp.router.e_score_correction_bias": "model-00076-of-00206.safetensors",
+ "model.layers.11.mlp.switch_mlp.down_proj.biases": "model-00076-of-00206.safetensors",
+ "model.layers.11.mlp.switch_mlp.down_proj.scales": "model-00075-of-00206.safetensors",
+ "model.layers.11.mlp.switch_mlp.down_proj.weight": "model-00075-of-00206.safetensors",
+ "model.layers.11.mlp.switch_mlp.gate_proj.biases": "model-00073-of-00206.safetensors",
+ "model.layers.11.mlp.switch_mlp.gate_proj.scales": "model-00072-of-00206.safetensors",
+ "model.layers.11.mlp.switch_mlp.gate_proj.weight": "model-00072-of-00206.safetensors",
+ "model.layers.11.mlp.switch_mlp.up_proj.biases": "model-00074-of-00206.safetensors",
+ "model.layers.11.mlp.switch_mlp.up_proj.scales": "model-00074-of-00206.safetensors",
+ "model.layers.11.mlp.switch_mlp.up_proj.weight": "model-00073-of-00206.safetensors",
+ "model.layers.11.mlps.0.down_proj.biases": "model-00076-of-00206.safetensors",
+ "model.layers.11.mlps.0.down_proj.scales": "model-00076-of-00206.safetensors",
+ "model.layers.11.mlps.0.down_proj.weight": "model-00076-of-00206.safetensors",
+ "model.layers.11.mlps.0.gate_proj.biases": "model-00076-of-00206.safetensors",
+ "model.layers.11.mlps.0.gate_proj.scales": "model-00076-of-00206.safetensors",
+ "model.layers.11.mlps.0.gate_proj.weight": "model-00076-of-00206.safetensors",
+ "model.layers.11.mlps.0.up_proj.biases": "model-00076-of-00206.safetensors",
+ "model.layers.11.mlps.0.up_proj.scales": "model-00076-of-00206.safetensors",
+ "model.layers.11.mlps.0.up_proj.weight": "model-00076-of-00206.safetensors",
+ "model.layers.11.mlps.1.down_proj.biases": "model-00076-of-00206.safetensors",
+ "model.layers.11.mlps.1.down_proj.scales": "model-00076-of-00206.safetensors",
+ "model.layers.11.mlps.1.down_proj.weight": "model-00076-of-00206.safetensors",
+ "model.layers.11.mlps.1.gate_proj.biases": "model-00076-of-00206.safetensors",
+ "model.layers.11.mlps.1.gate_proj.scales": "model-00076-of-00206.safetensors",
+ "model.layers.11.mlps.1.gate_proj.weight": "model-00076-of-00206.safetensors",
+ "model.layers.11.mlps.1.up_proj.biases": "model-00076-of-00206.safetensors",
+ "model.layers.11.mlps.1.up_proj.scales": "model-00076-of-00206.safetensors",
+ "model.layers.11.mlps.1.up_proj.weight": "model-00076-of-00206.safetensors",
+ "model.layers.11.post_attention_layernorm.0.weight": "model-00076-of-00206.safetensors",
+ "model.layers.11.post_attention_layernorm.1.weight": "model-00076-of-00206.safetensors",
+ "model.layers.11.self_attn.0.embed_q.biases": "model-00076-of-00206.safetensors",
+ "model.layers.11.self_attn.0.embed_q.scales": "model-00076-of-00206.safetensors",
+ "model.layers.11.self_attn.0.embed_q.weight": "model-00076-of-00206.safetensors",
+ "model.layers.11.self_attn.0.indexer.k_norm.weight": "model-00076-of-00206.safetensors",
+ "model.layers.11.self_attn.0.indexer.weights_proj.biases": "model-00076-of-00206.safetensors",
+ "model.layers.11.self_attn.0.indexer.weights_proj.scales": "model-00076-of-00206.safetensors",
+ "model.layers.11.self_attn.0.indexer.weights_proj.weight": "model-00076-of-00206.safetensors",
+ "model.layers.11.self_attn.0.indexer.wk.biases": "model-00076-of-00206.safetensors",
+ "model.layers.11.self_attn.0.indexer.wk.scales": "model-00076-of-00206.safetensors",
+ "model.layers.11.self_attn.0.indexer.wk.weight": "model-00076-of-00206.safetensors",
+ "model.layers.11.self_attn.0.indexer.wq_b.biases": "model-00076-of-00206.safetensors",
+ "model.layers.11.self_attn.0.indexer.wq_b.scales": "model-00076-of-00206.safetensors",
+ "model.layers.11.self_attn.0.indexer.wq_b.weight": "model-00076-of-00206.safetensors",
+ "model.layers.11.self_attn.0.kv_a_layernorm.weight": "model-00076-of-00206.safetensors",
+ "model.layers.11.self_attn.0.kv_a_proj_with_mqa.biases": "model-00076-of-00206.safetensors",
+ "model.layers.11.self_attn.0.kv_a_proj_with_mqa.scales": "model-00076-of-00206.safetensors",
+ "model.layers.11.self_attn.0.kv_a_proj_with_mqa.weight": "model-00076-of-00206.safetensors",
+ "model.layers.11.self_attn.0.o_proj.biases": "model-00076-of-00206.safetensors",
+ "model.layers.11.self_attn.0.o_proj.scales": "model-00076-of-00206.safetensors",
+ "model.layers.11.self_attn.0.o_proj.weight": "model-00076-of-00206.safetensors",
+ "model.layers.11.self_attn.0.q_a_layernorm.weight": "model-00076-of-00206.safetensors",
+ "model.layers.11.self_attn.0.q_a_proj.biases": "model-00076-of-00206.safetensors",
+ "model.layers.11.self_attn.0.q_a_proj.scales": "model-00076-of-00206.safetensors",
+ "model.layers.11.self_attn.0.q_a_proj.weight": "model-00076-of-00206.safetensors",
+ "model.layers.11.self_attn.0.q_b_proj.biases": "model-00076-of-00206.safetensors",
+ "model.layers.11.self_attn.0.q_b_proj.scales": "model-00076-of-00206.safetensors",
+ "model.layers.11.self_attn.0.q_b_proj.weight": "model-00076-of-00206.safetensors",
+ "model.layers.11.self_attn.0.unembed_out.biases": "model-00076-of-00206.safetensors",
+ "model.layers.11.self_attn.0.unembed_out.scales": "model-00076-of-00206.safetensors",
+ "model.layers.11.self_attn.0.unembed_out.weight": "model-00076-of-00206.safetensors",
+ "model.layers.11.self_attn.1.embed_q.biases": "model-00076-of-00206.safetensors",
+ "model.layers.11.self_attn.1.embed_q.scales": "model-00076-of-00206.safetensors",
+ "model.layers.11.self_attn.1.embed_q.weight": "model-00076-of-00206.safetensors",
+ "model.layers.11.self_attn.1.kv_a_layernorm.weight": "model-00076-of-00206.safetensors",
+ "model.layers.11.self_attn.1.kv_a_proj_with_mqa.biases": "model-00076-of-00206.safetensors",
+ "model.layers.11.self_attn.1.kv_a_proj_with_mqa.scales": "model-00076-of-00206.safetensors",
+ "model.layers.11.self_attn.1.kv_a_proj_with_mqa.weight": "model-00076-of-00206.safetensors",
+ "model.layers.11.self_attn.1.o_proj.biases": "model-00076-of-00206.safetensors",
+ "model.layers.11.self_attn.1.o_proj.scales": "model-00076-of-00206.safetensors",
+ "model.layers.11.self_attn.1.o_proj.weight": "model-00076-of-00206.safetensors",
+ "model.layers.11.self_attn.1.q_a_layernorm.weight": "model-00076-of-00206.safetensors",
+ "model.layers.11.self_attn.1.q_a_proj.biases": "model-00076-of-00206.safetensors",
+ "model.layers.11.self_attn.1.q_a_proj.scales": "model-00076-of-00206.safetensors",
+ "model.layers.11.self_attn.1.q_a_proj.weight": "model-00076-of-00206.safetensors",
+ "model.layers.11.self_attn.1.q_b_proj.biases": "model-00076-of-00206.safetensors",
+ "model.layers.11.self_attn.1.q_b_proj.scales": "model-00076-of-00206.safetensors",
+ "model.layers.11.self_attn.1.q_b_proj.weight": "model-00076-of-00206.safetensors",
+ "model.layers.11.self_attn.1.unembed_out.biases": "model-00076-of-00206.safetensors",
+ "model.layers.11.self_attn.1.unembed_out.scales": "model-00076-of-00206.safetensors",
+ "model.layers.11.self_attn.1.unembed_out.weight": "model-00076-of-00206.safetensors",
+ "model.layers.12.input_layernorm.0.weight": "model-00081-of-00206.safetensors",
+ "model.layers.12.input_layernorm.1.weight": "model-00081-of-00206.safetensors",
+ "model.layers.12.mlp.router.classifier.biases": "model-00081-of-00206.safetensors",
+ "model.layers.12.mlp.router.classifier.scales": "model-00081-of-00206.safetensors",
+ "model.layers.12.mlp.router.classifier.weight": "model-00081-of-00206.safetensors",
+ "model.layers.12.mlp.router.e_score_correction_bias": "model-00081-of-00206.safetensors",
+ "model.layers.12.mlp.switch_mlp.down_proj.biases": "model-00081-of-00206.safetensors",
+ "model.layers.12.mlp.switch_mlp.down_proj.scales": "model-00080-of-00206.safetensors",
+ "model.layers.12.mlp.switch_mlp.down_proj.weight": "model-00080-of-00206.safetensors",
+ "model.layers.12.mlp.switch_mlp.gate_proj.biases": "model-00078-of-00206.safetensors",
+ "model.layers.12.mlp.switch_mlp.gate_proj.scales": "model-00077-of-00206.safetensors",
+ "model.layers.12.mlp.switch_mlp.gate_proj.weight": "model-00077-of-00206.safetensors",
+ "model.layers.12.mlp.switch_mlp.up_proj.biases": "model-00079-of-00206.safetensors",
+ "model.layers.12.mlp.switch_mlp.up_proj.scales": "model-00079-of-00206.safetensors",
+ "model.layers.12.mlp.switch_mlp.up_proj.weight": "model-00078-of-00206.safetensors",
+ "model.layers.12.mlps.0.down_proj.biases": "model-00081-of-00206.safetensors",
+ "model.layers.12.mlps.0.down_proj.scales": "model-00081-of-00206.safetensors",
+ "model.layers.12.mlps.0.down_proj.weight": "model-00081-of-00206.safetensors",
+ "model.layers.12.mlps.0.gate_proj.biases": "model-00081-of-00206.safetensors",
+ "model.layers.12.mlps.0.gate_proj.scales": "model-00081-of-00206.safetensors",
+ "model.layers.12.mlps.0.gate_proj.weight": "model-00081-of-00206.safetensors",
+ "model.layers.12.mlps.0.up_proj.biases": "model-00081-of-00206.safetensors",
+ "model.layers.12.mlps.0.up_proj.scales": "model-00081-of-00206.safetensors",
+ "model.layers.12.mlps.0.up_proj.weight": "model-00081-of-00206.safetensors",
+ "model.layers.12.mlps.1.down_proj.biases": "model-00081-of-00206.safetensors",
+ "model.layers.12.mlps.1.down_proj.scales": "model-00081-of-00206.safetensors",
+ "model.layers.12.mlps.1.down_proj.weight": "model-00081-of-00206.safetensors",
+ "model.layers.12.mlps.1.gate_proj.biases": "model-00081-of-00206.safetensors",
+ "model.layers.12.mlps.1.gate_proj.scales": "model-00081-of-00206.safetensors",
+ "model.layers.12.mlps.1.gate_proj.weight": "model-00081-of-00206.safetensors",
+ "model.layers.12.mlps.1.up_proj.biases": "model-00081-of-00206.safetensors",
+ "model.layers.12.mlps.1.up_proj.scales": "model-00081-of-00206.safetensors",
+ "model.layers.12.mlps.1.up_proj.weight": "model-00081-of-00206.safetensors",
+ "model.layers.12.post_attention_layernorm.0.weight": "model-00081-of-00206.safetensors",
+ "model.layers.12.post_attention_layernorm.1.weight": "model-00081-of-00206.safetensors",
+ "model.layers.12.self_attn.0.embed_q.biases": "model-00081-of-00206.safetensors",
+ "model.layers.12.self_attn.0.embed_q.scales": "model-00081-of-00206.safetensors",
+ "model.layers.12.self_attn.0.embed_q.weight": "model-00081-of-00206.safetensors",
+ "model.layers.12.self_attn.0.indexer.k_norm.weight": "model-00081-of-00206.safetensors",
+ "model.layers.12.self_attn.0.indexer.weights_proj.biases": "model-00081-of-00206.safetensors",
+ "model.layers.12.self_attn.0.indexer.weights_proj.scales": "model-00081-of-00206.safetensors",
+ "model.layers.12.self_attn.0.indexer.weights_proj.weight": "model-00081-of-00206.safetensors",
+ "model.layers.12.self_attn.0.indexer.wk.biases": "model-00081-of-00206.safetensors",
+ "model.layers.12.self_attn.0.indexer.wk.scales": "model-00081-of-00206.safetensors",
+ "model.layers.12.self_attn.0.indexer.wk.weight": "model-00081-of-00206.safetensors",
+ "model.layers.12.self_attn.0.indexer.wq_b.biases": "model-00081-of-00206.safetensors",
+ "model.layers.12.self_attn.0.indexer.wq_b.scales": "model-00081-of-00206.safetensors",
+ "model.layers.12.self_attn.0.indexer.wq_b.weight": "model-00081-of-00206.safetensors",
+ "model.layers.12.self_attn.0.kv_a_layernorm.weight": "model-00081-of-00206.safetensors",
+ "model.layers.12.self_attn.0.kv_a_proj_with_mqa.biases": "model-00081-of-00206.safetensors",
+ "model.layers.12.self_attn.0.kv_a_proj_with_mqa.scales": "model-00081-of-00206.safetensors",
+ "model.layers.12.self_attn.0.kv_a_proj_with_mqa.weight": "model-00081-of-00206.safetensors",
+ "model.layers.12.self_attn.0.o_proj.biases": "model-00081-of-00206.safetensors",
+ "model.layers.12.self_attn.0.o_proj.scales": "model-00081-of-00206.safetensors",
+ "model.layers.12.self_attn.0.o_proj.weight": "model-00081-of-00206.safetensors",
+ "model.layers.12.self_attn.0.q_a_layernorm.weight": "model-00081-of-00206.safetensors",
+ "model.layers.12.self_attn.0.q_a_proj.biases": "model-00081-of-00206.safetensors",
+ "model.layers.12.self_attn.0.q_a_proj.scales": "model-00081-of-00206.safetensors",
+ "model.layers.12.self_attn.0.q_a_proj.weight": "model-00081-of-00206.safetensors",
+ "model.layers.12.self_attn.0.q_b_proj.biases": "model-00081-of-00206.safetensors",
+ "model.layers.12.self_attn.0.q_b_proj.scales": "model-00081-of-00206.safetensors",
+ "model.layers.12.self_attn.0.q_b_proj.weight": "model-00081-of-00206.safetensors",
+ "model.layers.12.self_attn.0.unembed_out.biases": "model-00081-of-00206.safetensors",
+ "model.layers.12.self_attn.0.unembed_out.scales": "model-00081-of-00206.safetensors",
+ "model.layers.12.self_attn.0.unembed_out.weight": "model-00081-of-00206.safetensors",
+ "model.layers.12.self_attn.1.embed_q.biases": "model-00081-of-00206.safetensors",
+ "model.layers.12.self_attn.1.embed_q.scales": "model-00081-of-00206.safetensors",
+ "model.layers.12.self_attn.1.embed_q.weight": "model-00081-of-00206.safetensors",
+ "model.layers.12.self_attn.1.kv_a_layernorm.weight": "model-00081-of-00206.safetensors",
+ "model.layers.12.self_attn.1.kv_a_proj_with_mqa.biases": "model-00081-of-00206.safetensors",
+ "model.layers.12.self_attn.1.kv_a_proj_with_mqa.scales": "model-00081-of-00206.safetensors",
+ "model.layers.12.self_attn.1.kv_a_proj_with_mqa.weight": "model-00081-of-00206.safetensors",
+ "model.layers.12.self_attn.1.o_proj.biases": "model-00081-of-00206.safetensors",
+ "model.layers.12.self_attn.1.o_proj.scales": "model-00081-of-00206.safetensors",
+ "model.layers.12.self_attn.1.o_proj.weight": "model-00081-of-00206.safetensors",
+ "model.layers.12.self_attn.1.q_a_layernorm.weight": "model-00081-of-00206.safetensors",
+ "model.layers.12.self_attn.1.q_a_proj.biases": "model-00081-of-00206.safetensors",
+ "model.layers.12.self_attn.1.q_a_proj.scales": "model-00081-of-00206.safetensors",
+ "model.layers.12.self_attn.1.q_a_proj.weight": "model-00081-of-00206.safetensors",
+ "model.layers.12.self_attn.1.q_b_proj.biases": "model-00081-of-00206.safetensors",
+ "model.layers.12.self_attn.1.q_b_proj.scales": "model-00081-of-00206.safetensors",
+ "model.layers.12.self_attn.1.q_b_proj.weight": "model-00081-of-00206.safetensors",
+ "model.layers.12.self_attn.1.unembed_out.biases": "model-00081-of-00206.safetensors",
+ "model.layers.12.self_attn.1.unembed_out.scales": "model-00081-of-00206.safetensors",
+ "model.layers.12.self_attn.1.unembed_out.weight": "model-00081-of-00206.safetensors",
+ "model.layers.13.input_layernorm.0.weight": "model-00086-of-00206.safetensors",
+ "model.layers.13.input_layernorm.1.weight": "model-00086-of-00206.safetensors",
+ "model.layers.13.mlp.router.classifier.biases": "model-00086-of-00206.safetensors",
+ "model.layers.13.mlp.router.classifier.scales": "model-00086-of-00206.safetensors",
+ "model.layers.13.mlp.router.classifier.weight": "model-00086-of-00206.safetensors",
+ "model.layers.13.mlp.router.e_score_correction_bias": "model-00086-of-00206.safetensors",
+ "model.layers.13.mlp.switch_mlp.down_proj.biases": "model-00086-of-00206.safetensors",
+ "model.layers.13.mlp.switch_mlp.down_proj.scales": "model-00085-of-00206.safetensors",
+ "model.layers.13.mlp.switch_mlp.down_proj.weight": "model-00085-of-00206.safetensors",
+ "model.layers.13.mlp.switch_mlp.gate_proj.biases": "model-00083-of-00206.safetensors",
+ "model.layers.13.mlp.switch_mlp.gate_proj.scales": "model-00082-of-00206.safetensors",
+ "model.layers.13.mlp.switch_mlp.gate_proj.weight": "model-00082-of-00206.safetensors",
+ "model.layers.13.mlp.switch_mlp.up_proj.biases": "model-00084-of-00206.safetensors",
+ "model.layers.13.mlp.switch_mlp.up_proj.scales": "model-00084-of-00206.safetensors",
+ "model.layers.13.mlp.switch_mlp.up_proj.weight": "model-00083-of-00206.safetensors",
+ "model.layers.13.mlps.0.down_proj.biases": "model-00086-of-00206.safetensors",
+ "model.layers.13.mlps.0.down_proj.scales": "model-00086-of-00206.safetensors",
+ "model.layers.13.mlps.0.down_proj.weight": "model-00086-of-00206.safetensors",
+ "model.layers.13.mlps.0.gate_proj.biases": "model-00086-of-00206.safetensors",
+ "model.layers.13.mlps.0.gate_proj.scales": "model-00086-of-00206.safetensors",
+ "model.layers.13.mlps.0.gate_proj.weight": "model-00086-of-00206.safetensors",
+ "model.layers.13.mlps.0.up_proj.biases": "model-00086-of-00206.safetensors",
+ "model.layers.13.mlps.0.up_proj.scales": "model-00086-of-00206.safetensors",
+ "model.layers.13.mlps.0.up_proj.weight": "model-00086-of-00206.safetensors",
+ "model.layers.13.mlps.1.down_proj.biases": "model-00086-of-00206.safetensors",
+ "model.layers.13.mlps.1.down_proj.scales": "model-00086-of-00206.safetensors",
+ "model.layers.13.mlps.1.down_proj.weight": "model-00086-of-00206.safetensors",
+ "model.layers.13.mlps.1.gate_proj.biases": "model-00086-of-00206.safetensors",
+ "model.layers.13.mlps.1.gate_proj.scales": "model-00086-of-00206.safetensors",
+ "model.layers.13.mlps.1.gate_proj.weight": "model-00086-of-00206.safetensors",
+ "model.layers.13.mlps.1.up_proj.biases": "model-00086-of-00206.safetensors",
+ "model.layers.13.mlps.1.up_proj.scales": "model-00086-of-00206.safetensors",
+ "model.layers.13.mlps.1.up_proj.weight": "model-00086-of-00206.safetensors",
+ "model.layers.13.post_attention_layernorm.0.weight": "model-00086-of-00206.safetensors",
+ "model.layers.13.post_attention_layernorm.1.weight": "model-00086-of-00206.safetensors",
+ "model.layers.13.self_attn.0.embed_q.biases": "model-00086-of-00206.safetensors",
+ "model.layers.13.self_attn.0.embed_q.scales": "model-00086-of-00206.safetensors",
+ "model.layers.13.self_attn.0.embed_q.weight": "model-00086-of-00206.safetensors",
+ "model.layers.13.self_attn.0.indexer.k_norm.weight": "model-00086-of-00206.safetensors",
+ "model.layers.13.self_attn.0.indexer.weights_proj.biases": "model-00086-of-00206.safetensors",
+ "model.layers.13.self_attn.0.indexer.weights_proj.scales": "model-00086-of-00206.safetensors",
+ "model.layers.13.self_attn.0.indexer.weights_proj.weight": "model-00086-of-00206.safetensors",
+ "model.layers.13.self_attn.0.indexer.wk.biases": "model-00086-of-00206.safetensors",
+ "model.layers.13.self_attn.0.indexer.wk.scales": "model-00086-of-00206.safetensors",
+ "model.layers.13.self_attn.0.indexer.wk.weight": "model-00086-of-00206.safetensors",
+ "model.layers.13.self_attn.0.indexer.wq_b.biases": "model-00086-of-00206.safetensors",
+ "model.layers.13.self_attn.0.indexer.wq_b.scales": "model-00086-of-00206.safetensors",
+ "model.layers.13.self_attn.0.indexer.wq_b.weight": "model-00086-of-00206.safetensors",
+ "model.layers.13.self_attn.0.kv_a_layernorm.weight": "model-00086-of-00206.safetensors",
+ "model.layers.13.self_attn.0.kv_a_proj_with_mqa.biases": "model-00086-of-00206.safetensors",
+ "model.layers.13.self_attn.0.kv_a_proj_with_mqa.scales": "model-00086-of-00206.safetensors",
+ "model.layers.13.self_attn.0.kv_a_proj_with_mqa.weight": "model-00086-of-00206.safetensors",
+ "model.layers.13.self_attn.0.o_proj.biases": "model-00086-of-00206.safetensors",
+ "model.layers.13.self_attn.0.o_proj.scales": "model-00086-of-00206.safetensors",
+ "model.layers.13.self_attn.0.o_proj.weight": "model-00086-of-00206.safetensors",
+ "model.layers.13.self_attn.0.q_a_layernorm.weight": "model-00086-of-00206.safetensors",
+ "model.layers.13.self_attn.0.q_a_proj.biases": "model-00086-of-00206.safetensors",
+ "model.layers.13.self_attn.0.q_a_proj.scales": "model-00086-of-00206.safetensors",
+ "model.layers.13.self_attn.0.q_a_proj.weight": "model-00086-of-00206.safetensors",
+ "model.layers.13.self_attn.0.q_b_proj.biases": "model-00086-of-00206.safetensors",
+ "model.layers.13.self_attn.0.q_b_proj.scales": "model-00086-of-00206.safetensors",
+ "model.layers.13.self_attn.0.q_b_proj.weight": "model-00086-of-00206.safetensors",
+ "model.layers.13.self_attn.0.unembed_out.biases": "model-00086-of-00206.safetensors",
+ "model.layers.13.self_attn.0.unembed_out.scales": "model-00086-of-00206.safetensors",
+ "model.layers.13.self_attn.0.unembed_out.weight": "model-00086-of-00206.safetensors",
+ "model.layers.13.self_attn.1.embed_q.biases": "model-00086-of-00206.safetensors",
+ "model.layers.13.self_attn.1.embed_q.scales": "model-00086-of-00206.safetensors",
+ "model.layers.13.self_attn.1.embed_q.weight": "model-00086-of-00206.safetensors",
+ "model.layers.13.self_attn.1.kv_a_layernorm.weight": "model-00086-of-00206.safetensors",
+ "model.layers.13.self_attn.1.kv_a_proj_with_mqa.biases": "model-00086-of-00206.safetensors",
+ "model.layers.13.self_attn.1.kv_a_proj_with_mqa.scales": "model-00086-of-00206.safetensors",
+ "model.layers.13.self_attn.1.kv_a_proj_with_mqa.weight": "model-00086-of-00206.safetensors",
+ "model.layers.13.self_attn.1.o_proj.biases": "model-00086-of-00206.safetensors",
+ "model.layers.13.self_attn.1.o_proj.scales": "model-00086-of-00206.safetensors",
+ "model.layers.13.self_attn.1.o_proj.weight": "model-00086-of-00206.safetensors",
+ "model.layers.13.self_attn.1.q_a_layernorm.weight": "model-00086-of-00206.safetensors",
+ "model.layers.13.self_attn.1.q_a_proj.biases": "model-00086-of-00206.safetensors",
+ "model.layers.13.self_attn.1.q_a_proj.scales": "model-00086-of-00206.safetensors",
+ "model.layers.13.self_attn.1.q_a_proj.weight": "model-00086-of-00206.safetensors",
+ "model.layers.13.self_attn.1.q_b_proj.biases": "model-00086-of-00206.safetensors",
+ "model.layers.13.self_attn.1.q_b_proj.scales": "model-00086-of-00206.safetensors",
+ "model.layers.13.self_attn.1.q_b_proj.weight": "model-00086-of-00206.safetensors",
+ "model.layers.13.self_attn.1.unembed_out.biases": "model-00086-of-00206.safetensors",
+ "model.layers.13.self_attn.1.unembed_out.scales": "model-00086-of-00206.safetensors",
+ "model.layers.13.self_attn.1.unembed_out.weight": "model-00086-of-00206.safetensors",
+ "model.layers.14.input_layernorm.0.weight": "model-00091-of-00206.safetensors",
+ "model.layers.14.input_layernorm.1.weight": "model-00091-of-00206.safetensors",
+ "model.layers.14.mlp.router.classifier.biases": "model-00091-of-00206.safetensors",
+ "model.layers.14.mlp.router.classifier.scales": "model-00091-of-00206.safetensors",
+ "model.layers.14.mlp.router.classifier.weight": "model-00091-of-00206.safetensors",
+ "model.layers.14.mlp.router.e_score_correction_bias": "model-00091-of-00206.safetensors",
+ "model.layers.14.mlp.switch_mlp.down_proj.biases": "model-00091-of-00206.safetensors",
+ "model.layers.14.mlp.switch_mlp.down_proj.scales": "model-00090-of-00206.safetensors",
+ "model.layers.14.mlp.switch_mlp.down_proj.weight": "model-00090-of-00206.safetensors",
+ "model.layers.14.mlp.switch_mlp.gate_proj.biases": "model-00088-of-00206.safetensors",
+ "model.layers.14.mlp.switch_mlp.gate_proj.scales": "model-00087-of-00206.safetensors",
+ "model.layers.14.mlp.switch_mlp.gate_proj.weight": "model-00087-of-00206.safetensors",
+ "model.layers.14.mlp.switch_mlp.up_proj.biases": "model-00089-of-00206.safetensors",
+ "model.layers.14.mlp.switch_mlp.up_proj.scales": "model-00089-of-00206.safetensors",
+ "model.layers.14.mlp.switch_mlp.up_proj.weight": "model-00088-of-00206.safetensors",
+ "model.layers.14.mlps.0.down_proj.biases": "model-00091-of-00206.safetensors",
+ "model.layers.14.mlps.0.down_proj.scales": "model-00091-of-00206.safetensors",
+ "model.layers.14.mlps.0.down_proj.weight": "model-00091-of-00206.safetensors",
+ "model.layers.14.mlps.0.gate_proj.biases": "model-00091-of-00206.safetensors",
+ "model.layers.14.mlps.0.gate_proj.scales": "model-00091-of-00206.safetensors",
+ "model.layers.14.mlps.0.gate_proj.weight": "model-00091-of-00206.safetensors",
+ "model.layers.14.mlps.0.up_proj.biases": "model-00091-of-00206.safetensors",
+ "model.layers.14.mlps.0.up_proj.scales": "model-00091-of-00206.safetensors",
+ "model.layers.14.mlps.0.up_proj.weight": "model-00091-of-00206.safetensors",
+ "model.layers.14.mlps.1.down_proj.biases": "model-00091-of-00206.safetensors",
+ "model.layers.14.mlps.1.down_proj.scales": "model-00091-of-00206.safetensors",
+ "model.layers.14.mlps.1.down_proj.weight": "model-00091-of-00206.safetensors",
+ "model.layers.14.mlps.1.gate_proj.biases": "model-00091-of-00206.safetensors",
+ "model.layers.14.mlps.1.gate_proj.scales": "model-00091-of-00206.safetensors",
+ "model.layers.14.mlps.1.gate_proj.weight": "model-00091-of-00206.safetensors",
+ "model.layers.14.mlps.1.up_proj.biases": "model-00091-of-00206.safetensors",
+ "model.layers.14.mlps.1.up_proj.scales": "model-00091-of-00206.safetensors",
+ "model.layers.14.mlps.1.up_proj.weight": "model-00091-of-00206.safetensors",
+ "model.layers.14.post_attention_layernorm.0.weight": "model-00091-of-00206.safetensors",
+ "model.layers.14.post_attention_layernorm.1.weight": "model-00091-of-00206.safetensors",
+ "model.layers.14.self_attn.0.embed_q.biases": "model-00091-of-00206.safetensors",
+ "model.layers.14.self_attn.0.embed_q.scales": "model-00091-of-00206.safetensors",
+ "model.layers.14.self_attn.0.embed_q.weight": "model-00091-of-00206.safetensors",
+ "model.layers.14.self_attn.0.indexer.k_norm.weight": "model-00091-of-00206.safetensors",
+ "model.layers.14.self_attn.0.indexer.weights_proj.biases": "model-00091-of-00206.safetensors",
+ "model.layers.14.self_attn.0.indexer.weights_proj.scales": "model-00091-of-00206.safetensors",
+ "model.layers.14.self_attn.0.indexer.weights_proj.weight": "model-00091-of-00206.safetensors",
+ "model.layers.14.self_attn.0.indexer.wk.biases": "model-00091-of-00206.safetensors",
+ "model.layers.14.self_attn.0.indexer.wk.scales": "model-00091-of-00206.safetensors",
+ "model.layers.14.self_attn.0.indexer.wk.weight": "model-00091-of-00206.safetensors",
+ "model.layers.14.self_attn.0.indexer.wq_b.biases": "model-00091-of-00206.safetensors",
+ "model.layers.14.self_attn.0.indexer.wq_b.scales": "model-00091-of-00206.safetensors",
+ "model.layers.14.self_attn.0.indexer.wq_b.weight": "model-00091-of-00206.safetensors",
+ "model.layers.14.self_attn.0.kv_a_layernorm.weight": "model-00091-of-00206.safetensors",
+ "model.layers.14.self_attn.0.kv_a_proj_with_mqa.biases": "model-00091-of-00206.safetensors",
+ "model.layers.14.self_attn.0.kv_a_proj_with_mqa.scales": "model-00091-of-00206.safetensors",
+ "model.layers.14.self_attn.0.kv_a_proj_with_mqa.weight": "model-00091-of-00206.safetensors",
+ "model.layers.14.self_attn.0.o_proj.biases": "model-00091-of-00206.safetensors",
+ "model.layers.14.self_attn.0.o_proj.scales": "model-00091-of-00206.safetensors",
+ "model.layers.14.self_attn.0.o_proj.weight": "model-00091-of-00206.safetensors",
+ "model.layers.14.self_attn.0.q_a_layernorm.weight": "model-00091-of-00206.safetensors",
+ "model.layers.14.self_attn.0.q_a_proj.biases": "model-00091-of-00206.safetensors",
+ "model.layers.14.self_attn.0.q_a_proj.scales": "model-00091-of-00206.safetensors",
+ "model.layers.14.self_attn.0.q_a_proj.weight": "model-00091-of-00206.safetensors",
+ "model.layers.14.self_attn.0.q_b_proj.biases": "model-00091-of-00206.safetensors",
+ "model.layers.14.self_attn.0.q_b_proj.scales": "model-00091-of-00206.safetensors",
+ "model.layers.14.self_attn.0.q_b_proj.weight": "model-00091-of-00206.safetensors",
+ "model.layers.14.self_attn.0.unembed_out.biases": "model-00091-of-00206.safetensors",
+ "model.layers.14.self_attn.0.unembed_out.scales": "model-00091-of-00206.safetensors",
+ "model.layers.14.self_attn.0.unembed_out.weight": "model-00091-of-00206.safetensors",
+ "model.layers.14.self_attn.1.embed_q.biases": "model-00091-of-00206.safetensors",
+ "model.layers.14.self_attn.1.embed_q.scales": "model-00091-of-00206.safetensors",
+ "model.layers.14.self_attn.1.embed_q.weight": "model-00091-of-00206.safetensors",
+ "model.layers.14.self_attn.1.kv_a_layernorm.weight": "model-00091-of-00206.safetensors",
+ "model.layers.14.self_attn.1.kv_a_proj_with_mqa.biases": "model-00091-of-00206.safetensors",
+ "model.layers.14.self_attn.1.kv_a_proj_with_mqa.scales": "model-00091-of-00206.safetensors",
+ "model.layers.14.self_attn.1.kv_a_proj_with_mqa.weight": "model-00091-of-00206.safetensors",
+ "model.layers.14.self_attn.1.o_proj.biases": "model-00091-of-00206.safetensors",
+ "model.layers.14.self_attn.1.o_proj.scales": "model-00091-of-00206.safetensors",
+ "model.layers.14.self_attn.1.o_proj.weight": "model-00091-of-00206.safetensors",
+ "model.layers.14.self_attn.1.q_a_layernorm.weight": "model-00091-of-00206.safetensors",
+ "model.layers.14.self_attn.1.q_a_proj.biases": "model-00091-of-00206.safetensors",
+ "model.layers.14.self_attn.1.q_a_proj.scales": "model-00091-of-00206.safetensors",
+ "model.layers.14.self_attn.1.q_a_proj.weight": "model-00091-of-00206.safetensors",
+ "model.layers.14.self_attn.1.q_b_proj.biases": "model-00091-of-00206.safetensors",
+ "model.layers.14.self_attn.1.q_b_proj.scales": "model-00091-of-00206.safetensors",
+ "model.layers.14.self_attn.1.q_b_proj.weight": "model-00091-of-00206.safetensors",
+ "model.layers.14.self_attn.1.unembed_out.biases": "model-00091-of-00206.safetensors",
+ "model.layers.14.self_attn.1.unembed_out.scales": "model-00091-of-00206.safetensors",
+ "model.layers.14.self_attn.1.unembed_out.weight": "model-00091-of-00206.safetensors",
+ "model.layers.15.input_layernorm.0.weight": "model-00096-of-00206.safetensors",
+ "model.layers.15.input_layernorm.1.weight": "model-00096-of-00206.safetensors",
+ "model.layers.15.mlp.router.classifier.biases": "model-00096-of-00206.safetensors",
+ "model.layers.15.mlp.router.classifier.scales": "model-00096-of-00206.safetensors",
+ "model.layers.15.mlp.router.classifier.weight": "model-00096-of-00206.safetensors",
+ "model.layers.15.mlp.router.e_score_correction_bias": "model-00096-of-00206.safetensors",
+ "model.layers.15.mlp.switch_mlp.down_proj.biases": "model-00096-of-00206.safetensors",
+ "model.layers.15.mlp.switch_mlp.down_proj.scales": "model-00095-of-00206.safetensors",
+ "model.layers.15.mlp.switch_mlp.down_proj.weight": "model-00095-of-00206.safetensors",
+ "model.layers.15.mlp.switch_mlp.gate_proj.biases": "model-00093-of-00206.safetensors",
+ "model.layers.15.mlp.switch_mlp.gate_proj.scales": "model-00092-of-00206.safetensors",
+ "model.layers.15.mlp.switch_mlp.gate_proj.weight": "model-00092-of-00206.safetensors",
+ "model.layers.15.mlp.switch_mlp.up_proj.biases": "model-00094-of-00206.safetensors",
+ "model.layers.15.mlp.switch_mlp.up_proj.scales": "model-00094-of-00206.safetensors",
+ "model.layers.15.mlp.switch_mlp.up_proj.weight": "model-00093-of-00206.safetensors",
+ "model.layers.15.mlps.0.down_proj.biases": "model-00096-of-00206.safetensors",
+ "model.layers.15.mlps.0.down_proj.scales": "model-00096-of-00206.safetensors",
+ "model.layers.15.mlps.0.down_proj.weight": "model-00096-of-00206.safetensors",
+ "model.layers.15.mlps.0.gate_proj.biases": "model-00096-of-00206.safetensors",
+ "model.layers.15.mlps.0.gate_proj.scales": "model-00096-of-00206.safetensors",
+ "model.layers.15.mlps.0.gate_proj.weight": "model-00096-of-00206.safetensors",
+ "model.layers.15.mlps.0.up_proj.biases": "model-00096-of-00206.safetensors",
+ "model.layers.15.mlps.0.up_proj.scales": "model-00096-of-00206.safetensors",
+ "model.layers.15.mlps.0.up_proj.weight": "model-00096-of-00206.safetensors",
+ "model.layers.15.mlps.1.down_proj.biases": "model-00096-of-00206.safetensors",
+ "model.layers.15.mlps.1.down_proj.scales": "model-00096-of-00206.safetensors",
+ "model.layers.15.mlps.1.down_proj.weight": "model-00096-of-00206.safetensors",
+ "model.layers.15.mlps.1.gate_proj.biases": "model-00096-of-00206.safetensors",
+ "model.layers.15.mlps.1.gate_proj.scales": "model-00096-of-00206.safetensors",
+ "model.layers.15.mlps.1.gate_proj.weight": "model-00096-of-00206.safetensors",
+ "model.layers.15.mlps.1.up_proj.biases": "model-00096-of-00206.safetensors",
+ "model.layers.15.mlps.1.up_proj.scales": "model-00096-of-00206.safetensors",
+ "model.layers.15.mlps.1.up_proj.weight": "model-00096-of-00206.safetensors",
+ "model.layers.15.post_attention_layernorm.0.weight": "model-00096-of-00206.safetensors",
+ "model.layers.15.post_attention_layernorm.1.weight": "model-00096-of-00206.safetensors",
+ "model.layers.15.self_attn.0.embed_q.biases": "model-00096-of-00206.safetensors",
+ "model.layers.15.self_attn.0.embed_q.scales": "model-00096-of-00206.safetensors",
+ "model.layers.15.self_attn.0.embed_q.weight": "model-00096-of-00206.safetensors",
+ "model.layers.15.self_attn.0.indexer.k_norm.weight": "model-00096-of-00206.safetensors",
+ "model.layers.15.self_attn.0.indexer.weights_proj.biases": "model-00096-of-00206.safetensors",
+ "model.layers.15.self_attn.0.indexer.weights_proj.scales": "model-00096-of-00206.safetensors",
+ "model.layers.15.self_attn.0.indexer.weights_proj.weight": "model-00096-of-00206.safetensors",
+ "model.layers.15.self_attn.0.indexer.wk.biases": "model-00096-of-00206.safetensors",
+ "model.layers.15.self_attn.0.indexer.wk.scales": "model-00096-of-00206.safetensors",
+ "model.layers.15.self_attn.0.indexer.wk.weight": "model-00096-of-00206.safetensors",
+ "model.layers.15.self_attn.0.indexer.wq_b.biases": "model-00096-of-00206.safetensors",
+ "model.layers.15.self_attn.0.indexer.wq_b.scales": "model-00096-of-00206.safetensors",
+ "model.layers.15.self_attn.0.indexer.wq_b.weight": "model-00096-of-00206.safetensors",
+ "model.layers.15.self_attn.0.kv_a_layernorm.weight": "model-00096-of-00206.safetensors",
+ "model.layers.15.self_attn.0.kv_a_proj_with_mqa.biases": "model-00096-of-00206.safetensors",
+ "model.layers.15.self_attn.0.kv_a_proj_with_mqa.scales": "model-00096-of-00206.safetensors",
+ "model.layers.15.self_attn.0.kv_a_proj_with_mqa.weight": "model-00096-of-00206.safetensors",
+ "model.layers.15.self_attn.0.o_proj.biases": "model-00096-of-00206.safetensors",
+ "model.layers.15.self_attn.0.o_proj.scales": "model-00096-of-00206.safetensors",
+ "model.layers.15.self_attn.0.o_proj.weight": "model-00096-of-00206.safetensors",
+ "model.layers.15.self_attn.0.q_a_layernorm.weight": "model-00096-of-00206.safetensors",
+ "model.layers.15.self_attn.0.q_a_proj.biases": "model-00096-of-00206.safetensors",
+ "model.layers.15.self_attn.0.q_a_proj.scales": "model-00096-of-00206.safetensors",
+ "model.layers.15.self_attn.0.q_a_proj.weight": "model-00096-of-00206.safetensors",
+ "model.layers.15.self_attn.0.q_b_proj.biases": "model-00096-of-00206.safetensors",
+ "model.layers.15.self_attn.0.q_b_proj.scales": "model-00096-of-00206.safetensors",
+ "model.layers.15.self_attn.0.q_b_proj.weight": "model-00096-of-00206.safetensors",
+ "model.layers.15.self_attn.0.unembed_out.biases": "model-00096-of-00206.safetensors",
+ "model.layers.15.self_attn.0.unembed_out.scales": "model-00096-of-00206.safetensors",
+ "model.layers.15.self_attn.0.unembed_out.weight": "model-00096-of-00206.safetensors",
+ "model.layers.15.self_attn.1.embed_q.biases": "model-00096-of-00206.safetensors",
+ "model.layers.15.self_attn.1.embed_q.scales": "model-00096-of-00206.safetensors",
+ "model.layers.15.self_attn.1.embed_q.weight": "model-00096-of-00206.safetensors",
+ "model.layers.15.self_attn.1.kv_a_layernorm.weight": "model-00096-of-00206.safetensors",
+ "model.layers.15.self_attn.1.kv_a_proj_with_mqa.biases": "model-00096-of-00206.safetensors",
+ "model.layers.15.self_attn.1.kv_a_proj_with_mqa.scales": "model-00096-of-00206.safetensors",
+ "model.layers.15.self_attn.1.kv_a_proj_with_mqa.weight": "model-00096-of-00206.safetensors",
+ "model.layers.15.self_attn.1.o_proj.biases": "model-00096-of-00206.safetensors",
+ "model.layers.15.self_attn.1.o_proj.scales": "model-00096-of-00206.safetensors",
+ "model.layers.15.self_attn.1.o_proj.weight": "model-00096-of-00206.safetensors",
+ "model.layers.15.self_attn.1.q_a_layernorm.weight": "model-00096-of-00206.safetensors",
+ "model.layers.15.self_attn.1.q_a_proj.biases": "model-00096-of-00206.safetensors",
+ "model.layers.15.self_attn.1.q_a_proj.scales": "model-00096-of-00206.safetensors",
+ "model.layers.15.self_attn.1.q_a_proj.weight": "model-00096-of-00206.safetensors",
+ "model.layers.15.self_attn.1.q_b_proj.biases": "model-00096-of-00206.safetensors",
+ "model.layers.15.self_attn.1.q_b_proj.scales": "model-00096-of-00206.safetensors",
+ "model.layers.15.self_attn.1.q_b_proj.weight": "model-00096-of-00206.safetensors",
+ "model.layers.15.self_attn.1.unembed_out.biases": "model-00096-of-00206.safetensors",
+ "model.layers.15.self_attn.1.unembed_out.scales": "model-00096-of-00206.safetensors",
+ "model.layers.15.self_attn.1.unembed_out.weight": "model-00096-of-00206.safetensors",
+ "model.layers.16.input_layernorm.0.weight": "model-00101-of-00206.safetensors",
+ "model.layers.16.input_layernorm.1.weight": "model-00101-of-00206.safetensors",
+ "model.layers.16.mlp.router.classifier.biases": "model-00101-of-00206.safetensors",
+ "model.layers.16.mlp.router.classifier.scales": "model-00101-of-00206.safetensors",
+ "model.layers.16.mlp.router.classifier.weight": "model-00101-of-00206.safetensors",
+ "model.layers.16.mlp.router.e_score_correction_bias": "model-00101-of-00206.safetensors",
+ "model.layers.16.mlp.switch_mlp.down_proj.biases": "model-00101-of-00206.safetensors",
+ "model.layers.16.mlp.switch_mlp.down_proj.scales": "model-00100-of-00206.safetensors",
+ "model.layers.16.mlp.switch_mlp.down_proj.weight": "model-00100-of-00206.safetensors",
+ "model.layers.16.mlp.switch_mlp.gate_proj.biases": "model-00098-of-00206.safetensors",
+ "model.layers.16.mlp.switch_mlp.gate_proj.scales": "model-00097-of-00206.safetensors",
+ "model.layers.16.mlp.switch_mlp.gate_proj.weight": "model-00097-of-00206.safetensors",
+ "model.layers.16.mlp.switch_mlp.up_proj.biases": "model-00099-of-00206.safetensors",
+ "model.layers.16.mlp.switch_mlp.up_proj.scales": "model-00099-of-00206.safetensors",
+ "model.layers.16.mlp.switch_mlp.up_proj.weight": "model-00098-of-00206.safetensors",
+ "model.layers.16.mlps.0.down_proj.biases": "model-00101-of-00206.safetensors",
+ "model.layers.16.mlps.0.down_proj.scales": "model-00101-of-00206.safetensors",
+ "model.layers.16.mlps.0.down_proj.weight": "model-00101-of-00206.safetensors",
+ "model.layers.16.mlps.0.gate_proj.biases": "model-00101-of-00206.safetensors",
+ "model.layers.16.mlps.0.gate_proj.scales": "model-00101-of-00206.safetensors",
+ "model.layers.16.mlps.0.gate_proj.weight": "model-00101-of-00206.safetensors",
+ "model.layers.16.mlps.0.up_proj.biases": "model-00101-of-00206.safetensors",
+ "model.layers.16.mlps.0.up_proj.scales": "model-00101-of-00206.safetensors",
+ "model.layers.16.mlps.0.up_proj.weight": "model-00101-of-00206.safetensors",
+ "model.layers.16.mlps.1.down_proj.biases": "model-00101-of-00206.safetensors",
+ "model.layers.16.mlps.1.down_proj.scales": "model-00101-of-00206.safetensors",
+ "model.layers.16.mlps.1.down_proj.weight": "model-00101-of-00206.safetensors",
+ "model.layers.16.mlps.1.gate_proj.biases": "model-00101-of-00206.safetensors",
+ "model.layers.16.mlps.1.gate_proj.scales": "model-00101-of-00206.safetensors",
+ "model.layers.16.mlps.1.gate_proj.weight": "model-00101-of-00206.safetensors",
+ "model.layers.16.mlps.1.up_proj.biases": "model-00101-of-00206.safetensors",
+ "model.layers.16.mlps.1.up_proj.scales": "model-00101-of-00206.safetensors",
+ "model.layers.16.mlps.1.up_proj.weight": "model-00101-of-00206.safetensors",
+ "model.layers.16.post_attention_layernorm.0.weight": "model-00101-of-00206.safetensors",
+ "model.layers.16.post_attention_layernorm.1.weight": "model-00101-of-00206.safetensors",
+ "model.layers.16.self_attn.0.embed_q.biases": "model-00101-of-00206.safetensors",
+ "model.layers.16.self_attn.0.embed_q.scales": "model-00101-of-00206.safetensors",
+ "model.layers.16.self_attn.0.embed_q.weight": "model-00101-of-00206.safetensors",
+ "model.layers.16.self_attn.0.indexer.k_norm.weight": "model-00101-of-00206.safetensors",
+ "model.layers.16.self_attn.0.indexer.weights_proj.biases": "model-00101-of-00206.safetensors",
+ "model.layers.16.self_attn.0.indexer.weights_proj.scales": "model-00101-of-00206.safetensors",
+ "model.layers.16.self_attn.0.indexer.weights_proj.weight": "model-00101-of-00206.safetensors",
+ "model.layers.16.self_attn.0.indexer.wk.biases": "model-00101-of-00206.safetensors",
+ "model.layers.16.self_attn.0.indexer.wk.scales": "model-00101-of-00206.safetensors",
+ "model.layers.16.self_attn.0.indexer.wk.weight": "model-00101-of-00206.safetensors",
+ "model.layers.16.self_attn.0.indexer.wq_b.biases": "model-00101-of-00206.safetensors",
+ "model.layers.16.self_attn.0.indexer.wq_b.scales": "model-00101-of-00206.safetensors",
+ "model.layers.16.self_attn.0.indexer.wq_b.weight": "model-00101-of-00206.safetensors",
+ "model.layers.16.self_attn.0.kv_a_layernorm.weight": "model-00101-of-00206.safetensors",
+ "model.layers.16.self_attn.0.kv_a_proj_with_mqa.biases": "model-00101-of-00206.safetensors",
+ "model.layers.16.self_attn.0.kv_a_proj_with_mqa.scales": "model-00101-of-00206.safetensors",
+ "model.layers.16.self_attn.0.kv_a_proj_with_mqa.weight": "model-00101-of-00206.safetensors",
+ "model.layers.16.self_attn.0.o_proj.biases": "model-00101-of-00206.safetensors",
+ "model.layers.16.self_attn.0.o_proj.scales": "model-00101-of-00206.safetensors",
+ "model.layers.16.self_attn.0.o_proj.weight": "model-00101-of-00206.safetensors",
+ "model.layers.16.self_attn.0.q_a_layernorm.weight": "model-00101-of-00206.safetensors",
+ "model.layers.16.self_attn.0.q_a_proj.biases": "model-00101-of-00206.safetensors",
+ "model.layers.16.self_attn.0.q_a_proj.scales": "model-00101-of-00206.safetensors",
+ "model.layers.16.self_attn.0.q_a_proj.weight": "model-00101-of-00206.safetensors",
+ "model.layers.16.self_attn.0.q_b_proj.biases": "model-00101-of-00206.safetensors",
+ "model.layers.16.self_attn.0.q_b_proj.scales": "model-00101-of-00206.safetensors",
+ "model.layers.16.self_attn.0.q_b_proj.weight": "model-00101-of-00206.safetensors",
+ "model.layers.16.self_attn.0.unembed_out.biases": "model-00101-of-00206.safetensors",
+ "model.layers.16.self_attn.0.unembed_out.scales": "model-00101-of-00206.safetensors",
+ "model.layers.16.self_attn.0.unembed_out.weight": "model-00101-of-00206.safetensors",
+ "model.layers.16.self_attn.1.embed_q.biases": "model-00101-of-00206.safetensors",
+ "model.layers.16.self_attn.1.embed_q.scales": "model-00101-of-00206.safetensors",
+ "model.layers.16.self_attn.1.embed_q.weight": "model-00101-of-00206.safetensors",
+ "model.layers.16.self_attn.1.kv_a_layernorm.weight": "model-00101-of-00206.safetensors",
+ "model.layers.16.self_attn.1.kv_a_proj_with_mqa.biases": "model-00101-of-00206.safetensors",
+ "model.layers.16.self_attn.1.kv_a_proj_with_mqa.scales": "model-00101-of-00206.safetensors",
+ "model.layers.16.self_attn.1.kv_a_proj_with_mqa.weight": "model-00101-of-00206.safetensors",
+ "model.layers.16.self_attn.1.o_proj.biases": "model-00101-of-00206.safetensors",
+ "model.layers.16.self_attn.1.o_proj.scales": "model-00101-of-00206.safetensors",
+ "model.layers.16.self_attn.1.o_proj.weight": "model-00101-of-00206.safetensors",
+ "model.layers.16.self_attn.1.q_a_layernorm.weight": "model-00101-of-00206.safetensors",
+ "model.layers.16.self_attn.1.q_a_proj.biases": "model-00101-of-00206.safetensors",
+ "model.layers.16.self_attn.1.q_a_proj.scales": "model-00101-of-00206.safetensors",
+ "model.layers.16.self_attn.1.q_a_proj.weight": "model-00101-of-00206.safetensors",
+ "model.layers.16.self_attn.1.q_b_proj.biases": "model-00101-of-00206.safetensors",
+ "model.layers.16.self_attn.1.q_b_proj.scales": "model-00101-of-00206.safetensors",
+ "model.layers.16.self_attn.1.q_b_proj.weight": "model-00101-of-00206.safetensors",
+ "model.layers.16.self_attn.1.unembed_out.biases": "model-00101-of-00206.safetensors",
+ "model.layers.16.self_attn.1.unembed_out.scales": "model-00101-of-00206.safetensors",
+ "model.layers.16.self_attn.1.unembed_out.weight": "model-00101-of-00206.safetensors",
+ "model.layers.17.input_layernorm.0.weight": "model-00106-of-00206.safetensors",
+ "model.layers.17.input_layernorm.1.weight": "model-00106-of-00206.safetensors",
+ "model.layers.17.mlp.router.classifier.biases": "model-00106-of-00206.safetensors",
+ "model.layers.17.mlp.router.classifier.scales": "model-00106-of-00206.safetensors",
+ "model.layers.17.mlp.router.classifier.weight": "model-00106-of-00206.safetensors",
+ "model.layers.17.mlp.router.e_score_correction_bias": "model-00106-of-00206.safetensors",
+ "model.layers.17.mlp.switch_mlp.down_proj.biases": "model-00106-of-00206.safetensors",
+ "model.layers.17.mlp.switch_mlp.down_proj.scales": "model-00105-of-00206.safetensors",
+ "model.layers.17.mlp.switch_mlp.down_proj.weight": "model-00105-of-00206.safetensors",
+ "model.layers.17.mlp.switch_mlp.gate_proj.biases": "model-00103-of-00206.safetensors",
+ "model.layers.17.mlp.switch_mlp.gate_proj.scales": "model-00102-of-00206.safetensors",
+ "model.layers.17.mlp.switch_mlp.gate_proj.weight": "model-00102-of-00206.safetensors",
+ "model.layers.17.mlp.switch_mlp.up_proj.biases": "model-00104-of-00206.safetensors",
+ "model.layers.17.mlp.switch_mlp.up_proj.scales": "model-00104-of-00206.safetensors",
+ "model.layers.17.mlp.switch_mlp.up_proj.weight": "model-00103-of-00206.safetensors",
+ "model.layers.17.mlps.0.down_proj.biases": "model-00106-of-00206.safetensors",
+ "model.layers.17.mlps.0.down_proj.scales": "model-00106-of-00206.safetensors",
+ "model.layers.17.mlps.0.down_proj.weight": "model-00106-of-00206.safetensors",
+ "model.layers.17.mlps.0.gate_proj.biases": "model-00106-of-00206.safetensors",
+ "model.layers.17.mlps.0.gate_proj.scales": "model-00106-of-00206.safetensors",
+ "model.layers.17.mlps.0.gate_proj.weight": "model-00106-of-00206.safetensors",
+ "model.layers.17.mlps.0.up_proj.biases": "model-00106-of-00206.safetensors",
+ "model.layers.17.mlps.0.up_proj.scales": "model-00106-of-00206.safetensors",
+ "model.layers.17.mlps.0.up_proj.weight": "model-00106-of-00206.safetensors",
+ "model.layers.17.mlps.1.down_proj.biases": "model-00106-of-00206.safetensors",
+ "model.layers.17.mlps.1.down_proj.scales": "model-00106-of-00206.safetensors",
+ "model.layers.17.mlps.1.down_proj.weight": "model-00106-of-00206.safetensors",
+ "model.layers.17.mlps.1.gate_proj.biases": "model-00106-of-00206.safetensors",
+ "model.layers.17.mlps.1.gate_proj.scales": "model-00106-of-00206.safetensors",
+ "model.layers.17.mlps.1.gate_proj.weight": "model-00106-of-00206.safetensors",
+ "model.layers.17.mlps.1.up_proj.biases": "model-00106-of-00206.safetensors",
+ "model.layers.17.mlps.1.up_proj.scales": "model-00106-of-00206.safetensors",
+ "model.layers.17.mlps.1.up_proj.weight": "model-00106-of-00206.safetensors",
+ "model.layers.17.post_attention_layernorm.0.weight": "model-00106-of-00206.safetensors",
+ "model.layers.17.post_attention_layernorm.1.weight": "model-00106-of-00206.safetensors",
+ "model.layers.17.self_attn.0.embed_q.biases": "model-00106-of-00206.safetensors",
+ "model.layers.17.self_attn.0.embed_q.scales": "model-00106-of-00206.safetensors",
+ "model.layers.17.self_attn.0.embed_q.weight": "model-00106-of-00206.safetensors",
+ "model.layers.17.self_attn.0.indexer.k_norm.weight": "model-00106-of-00206.safetensors",
+ "model.layers.17.self_attn.0.indexer.weights_proj.biases": "model-00106-of-00206.safetensors",
+ "model.layers.17.self_attn.0.indexer.weights_proj.scales": "model-00106-of-00206.safetensors",
+ "model.layers.17.self_attn.0.indexer.weights_proj.weight": "model-00106-of-00206.safetensors",
+ "model.layers.17.self_attn.0.indexer.wk.biases": "model-00106-of-00206.safetensors",
+ "model.layers.17.self_attn.0.indexer.wk.scales": "model-00106-of-00206.safetensors",
+ "model.layers.17.self_attn.0.indexer.wk.weight": "model-00106-of-00206.safetensors",
+ "model.layers.17.self_attn.0.indexer.wq_b.biases": "model-00106-of-00206.safetensors",
+ "model.layers.17.self_attn.0.indexer.wq_b.scales": "model-00106-of-00206.safetensors",
+ "model.layers.17.self_attn.0.indexer.wq_b.weight": "model-00106-of-00206.safetensors",
+ "model.layers.17.self_attn.0.kv_a_layernorm.weight": "model-00106-of-00206.safetensors",
+ "model.layers.17.self_attn.0.kv_a_proj_with_mqa.biases": "model-00106-of-00206.safetensors",
+ "model.layers.17.self_attn.0.kv_a_proj_with_mqa.scales": "model-00106-of-00206.safetensors",
+ "model.layers.17.self_attn.0.kv_a_proj_with_mqa.weight": "model-00106-of-00206.safetensors",
+ "model.layers.17.self_attn.0.o_proj.biases": "model-00106-of-00206.safetensors",
+ "model.layers.17.self_attn.0.o_proj.scales": "model-00106-of-00206.safetensors",
+ "model.layers.17.self_attn.0.o_proj.weight": "model-00106-of-00206.safetensors",
+ "model.layers.17.self_attn.0.q_a_layernorm.weight": "model-00106-of-00206.safetensors",
+ "model.layers.17.self_attn.0.q_a_proj.biases": "model-00106-of-00206.safetensors",
+ "model.layers.17.self_attn.0.q_a_proj.scales": "model-00106-of-00206.safetensors",
+ "model.layers.17.self_attn.0.q_a_proj.weight": "model-00106-of-00206.safetensors",
+ "model.layers.17.self_attn.0.q_b_proj.biases": "model-00106-of-00206.safetensors",
+ "model.layers.17.self_attn.0.q_b_proj.scales": "model-00106-of-00206.safetensors",
+ "model.layers.17.self_attn.0.q_b_proj.weight": "model-00106-of-00206.safetensors",
+ "model.layers.17.self_attn.0.unembed_out.biases": "model-00106-of-00206.safetensors",
+ "model.layers.17.self_attn.0.unembed_out.scales": "model-00106-of-00206.safetensors",
+ "model.layers.17.self_attn.0.unembed_out.weight": "model-00106-of-00206.safetensors",
+ "model.layers.17.self_attn.1.embed_q.biases": "model-00106-of-00206.safetensors",
+ "model.layers.17.self_attn.1.embed_q.scales": "model-00106-of-00206.safetensors",
+ "model.layers.17.self_attn.1.embed_q.weight": "model-00106-of-00206.safetensors",
+ "model.layers.17.self_attn.1.kv_a_layernorm.weight": "model-00106-of-00206.safetensors",
+ "model.layers.17.self_attn.1.kv_a_proj_with_mqa.biases": "model-00106-of-00206.safetensors",
+ "model.layers.17.self_attn.1.kv_a_proj_with_mqa.scales": "model-00106-of-00206.safetensors",
+ "model.layers.17.self_attn.1.kv_a_proj_with_mqa.weight": "model-00106-of-00206.safetensors",
+ "model.layers.17.self_attn.1.o_proj.biases": "model-00106-of-00206.safetensors",
+ "model.layers.17.self_attn.1.o_proj.scales": "model-00106-of-00206.safetensors",
+ "model.layers.17.self_attn.1.o_proj.weight": "model-00106-of-00206.safetensors",
+ "model.layers.17.self_attn.1.q_a_layernorm.weight": "model-00106-of-00206.safetensors",
+ "model.layers.17.self_attn.1.q_a_proj.biases": "model-00106-of-00206.safetensors",
+ "model.layers.17.self_attn.1.q_a_proj.scales": "model-00106-of-00206.safetensors",
+ "model.layers.17.self_attn.1.q_a_proj.weight": "model-00106-of-00206.safetensors",
+ "model.layers.17.self_attn.1.q_b_proj.biases": "model-00106-of-00206.safetensors",
+ "model.layers.17.self_attn.1.q_b_proj.scales": "model-00106-of-00206.safetensors",
+ "model.layers.17.self_attn.1.q_b_proj.weight": "model-00106-of-00206.safetensors",
+ "model.layers.17.self_attn.1.unembed_out.biases": "model-00106-of-00206.safetensors",
+ "model.layers.17.self_attn.1.unembed_out.scales": "model-00106-of-00206.safetensors",
+ "model.layers.17.self_attn.1.unembed_out.weight": "model-00106-of-00206.safetensors",
+ "model.layers.18.input_layernorm.0.weight": "model-00111-of-00206.safetensors",
+ "model.layers.18.input_layernorm.1.weight": "model-00111-of-00206.safetensors",
+ "model.layers.18.mlp.router.classifier.biases": "model-00111-of-00206.safetensors",
+ "model.layers.18.mlp.router.classifier.scales": "model-00111-of-00206.safetensors",
+ "model.layers.18.mlp.router.classifier.weight": "model-00111-of-00206.safetensors",
+ "model.layers.18.mlp.router.e_score_correction_bias": "model-00111-of-00206.safetensors",
+ "model.layers.18.mlp.switch_mlp.down_proj.biases": "model-00111-of-00206.safetensors",
+ "model.layers.18.mlp.switch_mlp.down_proj.scales": "model-00110-of-00206.safetensors",
+ "model.layers.18.mlp.switch_mlp.down_proj.weight": "model-00110-of-00206.safetensors",
+ "model.layers.18.mlp.switch_mlp.gate_proj.biases": "model-00108-of-00206.safetensors",
+ "model.layers.18.mlp.switch_mlp.gate_proj.scales": "model-00107-of-00206.safetensors",
+ "model.layers.18.mlp.switch_mlp.gate_proj.weight": "model-00107-of-00206.safetensors",
+ "model.layers.18.mlp.switch_mlp.up_proj.biases": "model-00109-of-00206.safetensors",
+ "model.layers.18.mlp.switch_mlp.up_proj.scales": "model-00109-of-00206.safetensors",
+ "model.layers.18.mlp.switch_mlp.up_proj.weight": "model-00108-of-00206.safetensors",
+ "model.layers.18.mlps.0.down_proj.biases": "model-00111-of-00206.safetensors",
+ "model.layers.18.mlps.0.down_proj.scales": "model-00111-of-00206.safetensors",
+ "model.layers.18.mlps.0.down_proj.weight": "model-00111-of-00206.safetensors",
+ "model.layers.18.mlps.0.gate_proj.biases": "model-00111-of-00206.safetensors",
+ "model.layers.18.mlps.0.gate_proj.scales": "model-00111-of-00206.safetensors",
+ "model.layers.18.mlps.0.gate_proj.weight": "model-00111-of-00206.safetensors",
+ "model.layers.18.mlps.0.up_proj.biases": "model-00111-of-00206.safetensors",
+ "model.layers.18.mlps.0.up_proj.scales": "model-00111-of-00206.safetensors",
+ "model.layers.18.mlps.0.up_proj.weight": "model-00111-of-00206.safetensors",
+ "model.layers.18.mlps.1.down_proj.biases": "model-00111-of-00206.safetensors",
+ "model.layers.18.mlps.1.down_proj.scales": "model-00111-of-00206.safetensors",
+ "model.layers.18.mlps.1.down_proj.weight": "model-00111-of-00206.safetensors",
+ "model.layers.18.mlps.1.gate_proj.biases": "model-00111-of-00206.safetensors",
+ "model.layers.18.mlps.1.gate_proj.scales": "model-00111-of-00206.safetensors",
+ "model.layers.18.mlps.1.gate_proj.weight": "model-00111-of-00206.safetensors",
+ "model.layers.18.mlps.1.up_proj.biases": "model-00111-of-00206.safetensors",
+ "model.layers.18.mlps.1.up_proj.scales": "model-00111-of-00206.safetensors",
+ "model.layers.18.mlps.1.up_proj.weight": "model-00111-of-00206.safetensors",
+ "model.layers.18.post_attention_layernorm.0.weight": "model-00111-of-00206.safetensors",
+ "model.layers.18.post_attention_layernorm.1.weight": "model-00111-of-00206.safetensors",
+ "model.layers.18.self_attn.0.embed_q.biases": "model-00111-of-00206.safetensors",
+ "model.layers.18.self_attn.0.embed_q.scales": "model-00111-of-00206.safetensors",
+ "model.layers.18.self_attn.0.embed_q.weight": "model-00111-of-00206.safetensors",
+ "model.layers.18.self_attn.0.indexer.k_norm.weight": "model-00111-of-00206.safetensors",
+ "model.layers.18.self_attn.0.indexer.weights_proj.biases": "model-00111-of-00206.safetensors",
+ "model.layers.18.self_attn.0.indexer.weights_proj.scales": "model-00111-of-00206.safetensors",
+ "model.layers.18.self_attn.0.indexer.weights_proj.weight": "model-00111-of-00206.safetensors",
+ "model.layers.18.self_attn.0.indexer.wk.biases": "model-00111-of-00206.safetensors",
+ "model.layers.18.self_attn.0.indexer.wk.scales": "model-00111-of-00206.safetensors",
+ "model.layers.18.self_attn.0.indexer.wk.weight": "model-00111-of-00206.safetensors",
+ "model.layers.18.self_attn.0.indexer.wq_b.biases": "model-00111-of-00206.safetensors",
+ "model.layers.18.self_attn.0.indexer.wq_b.scales": "model-00111-of-00206.safetensors",
+ "model.layers.18.self_attn.0.indexer.wq_b.weight": "model-00111-of-00206.safetensors",
+ "model.layers.18.self_attn.0.kv_a_layernorm.weight": "model-00111-of-00206.safetensors",
+ "model.layers.18.self_attn.0.kv_a_proj_with_mqa.biases": "model-00111-of-00206.safetensors",
+ "model.layers.18.self_attn.0.kv_a_proj_with_mqa.scales": "model-00111-of-00206.safetensors",
+ "model.layers.18.self_attn.0.kv_a_proj_with_mqa.weight": "model-00111-of-00206.safetensors",
+ "model.layers.18.self_attn.0.o_proj.biases": "model-00111-of-00206.safetensors",
+ "model.layers.18.self_attn.0.o_proj.scales": "model-00111-of-00206.safetensors",
+ "model.layers.18.self_attn.0.o_proj.weight": "model-00111-of-00206.safetensors",
+ "model.layers.18.self_attn.0.q_a_layernorm.weight": "model-00111-of-00206.safetensors",
+ "model.layers.18.self_attn.0.q_a_proj.biases": "model-00111-of-00206.safetensors",
+ "model.layers.18.self_attn.0.q_a_proj.scales": "model-00111-of-00206.safetensors",
+ "model.layers.18.self_attn.0.q_a_proj.weight": "model-00111-of-00206.safetensors",
+ "model.layers.18.self_attn.0.q_b_proj.biases": "model-00111-of-00206.safetensors",
+ "model.layers.18.self_attn.0.q_b_proj.scales": "model-00111-of-00206.safetensors",
+ "model.layers.18.self_attn.0.q_b_proj.weight": "model-00111-of-00206.safetensors",
+ "model.layers.18.self_attn.0.unembed_out.biases": "model-00111-of-00206.safetensors",
+ "model.layers.18.self_attn.0.unembed_out.scales": "model-00111-of-00206.safetensors",
+ "model.layers.18.self_attn.0.unembed_out.weight": "model-00111-of-00206.safetensors",
+ "model.layers.18.self_attn.1.embed_q.biases": "model-00111-of-00206.safetensors",
+ "model.layers.18.self_attn.1.embed_q.scales": "model-00111-of-00206.safetensors",
+ "model.layers.18.self_attn.1.embed_q.weight": "model-00111-of-00206.safetensors",
+ "model.layers.18.self_attn.1.kv_a_layernorm.weight": "model-00111-of-00206.safetensors",
+ "model.layers.18.self_attn.1.kv_a_proj_with_mqa.biases": "model-00111-of-00206.safetensors",
+ "model.layers.18.self_attn.1.kv_a_proj_with_mqa.scales": "model-00111-of-00206.safetensors",
+ "model.layers.18.self_attn.1.kv_a_proj_with_mqa.weight": "model-00111-of-00206.safetensors",
+ "model.layers.18.self_attn.1.o_proj.biases": "model-00111-of-00206.safetensors",
+ "model.layers.18.self_attn.1.o_proj.scales": "model-00111-of-00206.safetensors",
+ "model.layers.18.self_attn.1.o_proj.weight": "model-00111-of-00206.safetensors",
+ "model.layers.18.self_attn.1.q_a_layernorm.weight": "model-00111-of-00206.safetensors",
+ "model.layers.18.self_attn.1.q_a_proj.biases": "model-00111-of-00206.safetensors",
+ "model.layers.18.self_attn.1.q_a_proj.scales": "model-00111-of-00206.safetensors",
+ "model.layers.18.self_attn.1.q_a_proj.weight": "model-00111-of-00206.safetensors",
+ "model.layers.18.self_attn.1.q_b_proj.biases": "model-00111-of-00206.safetensors",
+ "model.layers.18.self_attn.1.q_b_proj.scales": "model-00111-of-00206.safetensors",
+ "model.layers.18.self_attn.1.q_b_proj.weight": "model-00111-of-00206.safetensors",
+ "model.layers.18.self_attn.1.unembed_out.biases": "model-00111-of-00206.safetensors",
+ "model.layers.18.self_attn.1.unembed_out.scales": "model-00111-of-00206.safetensors",
+ "model.layers.18.self_attn.1.unembed_out.weight": "model-00111-of-00206.safetensors",
+ "model.layers.19.input_layernorm.0.weight": "model-00116-of-00206.safetensors",
+ "model.layers.19.input_layernorm.1.weight": "model-00116-of-00206.safetensors",
+ "model.layers.19.mlp.router.classifier.biases": "model-00116-of-00206.safetensors",
+ "model.layers.19.mlp.router.classifier.scales": "model-00116-of-00206.safetensors",
+ "model.layers.19.mlp.router.classifier.weight": "model-00116-of-00206.safetensors",
+ "model.layers.19.mlp.router.e_score_correction_bias": "model-00116-of-00206.safetensors",
+ "model.layers.19.mlp.switch_mlp.down_proj.biases": "model-00116-of-00206.safetensors",
+ "model.layers.19.mlp.switch_mlp.down_proj.scales": "model-00115-of-00206.safetensors",
+ "model.layers.19.mlp.switch_mlp.down_proj.weight": "model-00115-of-00206.safetensors",
+ "model.layers.19.mlp.switch_mlp.gate_proj.biases": "model-00113-of-00206.safetensors",
+ "model.layers.19.mlp.switch_mlp.gate_proj.scales": "model-00112-of-00206.safetensors",
+ "model.layers.19.mlp.switch_mlp.gate_proj.weight": "model-00112-of-00206.safetensors",
+ "model.layers.19.mlp.switch_mlp.up_proj.biases": "model-00114-of-00206.safetensors",
+ "model.layers.19.mlp.switch_mlp.up_proj.scales": "model-00114-of-00206.safetensors",
+ "model.layers.19.mlp.switch_mlp.up_proj.weight": "model-00113-of-00206.safetensors",
+ "model.layers.19.mlps.0.down_proj.biases": "model-00116-of-00206.safetensors",
+ "model.layers.19.mlps.0.down_proj.scales": "model-00116-of-00206.safetensors",
+ "model.layers.19.mlps.0.down_proj.weight": "model-00116-of-00206.safetensors",
+ "model.layers.19.mlps.0.gate_proj.biases": "model-00116-of-00206.safetensors",
+ "model.layers.19.mlps.0.gate_proj.scales": "model-00116-of-00206.safetensors",
+ "model.layers.19.mlps.0.gate_proj.weight": "model-00116-of-00206.safetensors",
+ "model.layers.19.mlps.0.up_proj.biases": "model-00116-of-00206.safetensors",
+ "model.layers.19.mlps.0.up_proj.scales": "model-00116-of-00206.safetensors",
+ "model.layers.19.mlps.0.up_proj.weight": "model-00116-of-00206.safetensors",
+ "model.layers.19.mlps.1.down_proj.biases": "model-00116-of-00206.safetensors",
+ "model.layers.19.mlps.1.down_proj.scales": "model-00116-of-00206.safetensors",
+ "model.layers.19.mlps.1.down_proj.weight": "model-00116-of-00206.safetensors",
+ "model.layers.19.mlps.1.gate_proj.biases": "model-00116-of-00206.safetensors",
+ "model.layers.19.mlps.1.gate_proj.scales": "model-00116-of-00206.safetensors",
+ "model.layers.19.mlps.1.gate_proj.weight": "model-00116-of-00206.safetensors",
+ "model.layers.19.mlps.1.up_proj.biases": "model-00116-of-00206.safetensors",
+ "model.layers.19.mlps.1.up_proj.scales": "model-00116-of-00206.safetensors",
+ "model.layers.19.mlps.1.up_proj.weight": "model-00116-of-00206.safetensors",
+ "model.layers.19.post_attention_layernorm.0.weight": "model-00116-of-00206.safetensors",
+ "model.layers.19.post_attention_layernorm.1.weight": "model-00116-of-00206.safetensors",
+ "model.layers.19.self_attn.0.embed_q.biases": "model-00116-of-00206.safetensors",
+ "model.layers.19.self_attn.0.embed_q.scales": "model-00116-of-00206.safetensors",
+ "model.layers.19.self_attn.0.embed_q.weight": "model-00116-of-00206.safetensors",
+ "model.layers.19.self_attn.0.indexer.k_norm.weight": "model-00116-of-00206.safetensors",
+ "model.layers.19.self_attn.0.indexer.weights_proj.biases": "model-00116-of-00206.safetensors",
+ "model.layers.19.self_attn.0.indexer.weights_proj.scales": "model-00116-of-00206.safetensors",
+ "model.layers.19.self_attn.0.indexer.weights_proj.weight": "model-00116-of-00206.safetensors",
+ "model.layers.19.self_attn.0.indexer.wk.biases": "model-00116-of-00206.safetensors",
+ "model.layers.19.self_attn.0.indexer.wk.scales": "model-00116-of-00206.safetensors",
+ "model.layers.19.self_attn.0.indexer.wk.weight": "model-00116-of-00206.safetensors",
+ "model.layers.19.self_attn.0.indexer.wq_b.biases": "model-00116-of-00206.safetensors",
+ "model.layers.19.self_attn.0.indexer.wq_b.scales": "model-00116-of-00206.safetensors",
+ "model.layers.19.self_attn.0.indexer.wq_b.weight": "model-00116-of-00206.safetensors",
+ "model.layers.19.self_attn.0.kv_a_layernorm.weight": "model-00116-of-00206.safetensors",
+ "model.layers.19.self_attn.0.kv_a_proj_with_mqa.biases": "model-00116-of-00206.safetensors",
+ "model.layers.19.self_attn.0.kv_a_proj_with_mqa.scales": "model-00116-of-00206.safetensors",
+ "model.layers.19.self_attn.0.kv_a_proj_with_mqa.weight": "model-00116-of-00206.safetensors",
+ "model.layers.19.self_attn.0.o_proj.biases": "model-00116-of-00206.safetensors",
+ "model.layers.19.self_attn.0.o_proj.scales": "model-00116-of-00206.safetensors",
+ "model.layers.19.self_attn.0.o_proj.weight": "model-00116-of-00206.safetensors",
+ "model.layers.19.self_attn.0.q_a_layernorm.weight": "model-00116-of-00206.safetensors",
+ "model.layers.19.self_attn.0.q_a_proj.biases": "model-00116-of-00206.safetensors",
+ "model.layers.19.self_attn.0.q_a_proj.scales": "model-00116-of-00206.safetensors",
+ "model.layers.19.self_attn.0.q_a_proj.weight": "model-00116-of-00206.safetensors",
+ "model.layers.19.self_attn.0.q_b_proj.biases": "model-00116-of-00206.safetensors",
+ "model.layers.19.self_attn.0.q_b_proj.scales": "model-00116-of-00206.safetensors",
+ "model.layers.19.self_attn.0.q_b_proj.weight": "model-00116-of-00206.safetensors",
+ "model.layers.19.self_attn.0.unembed_out.biases": "model-00116-of-00206.safetensors",
+ "model.layers.19.self_attn.0.unembed_out.scales": "model-00116-of-00206.safetensors",
+ "model.layers.19.self_attn.0.unembed_out.weight": "model-00116-of-00206.safetensors",
+ "model.layers.19.self_attn.1.embed_q.biases": "model-00116-of-00206.safetensors",
+ "model.layers.19.self_attn.1.embed_q.scales": "model-00116-of-00206.safetensors",
+ "model.layers.19.self_attn.1.embed_q.weight": "model-00116-of-00206.safetensors",
+ "model.layers.19.self_attn.1.kv_a_layernorm.weight": "model-00116-of-00206.safetensors",
+ "model.layers.19.self_attn.1.kv_a_proj_with_mqa.biases": "model-00116-of-00206.safetensors",
+ "model.layers.19.self_attn.1.kv_a_proj_with_mqa.scales": "model-00116-of-00206.safetensors",
+ "model.layers.19.self_attn.1.kv_a_proj_with_mqa.weight": "model-00116-of-00206.safetensors",
+ "model.layers.19.self_attn.1.o_proj.biases": "model-00116-of-00206.safetensors",
+ "model.layers.19.self_attn.1.o_proj.scales": "model-00116-of-00206.safetensors",
+ "model.layers.19.self_attn.1.o_proj.weight": "model-00116-of-00206.safetensors",
+ "model.layers.19.self_attn.1.q_a_layernorm.weight": "model-00116-of-00206.safetensors",
+ "model.layers.19.self_attn.1.q_a_proj.biases": "model-00116-of-00206.safetensors",
+ "model.layers.19.self_attn.1.q_a_proj.scales": "model-00116-of-00206.safetensors",
+ "model.layers.19.self_attn.1.q_a_proj.weight": "model-00116-of-00206.safetensors",
+ "model.layers.19.self_attn.1.q_b_proj.biases": "model-00116-of-00206.safetensors",
+ "model.layers.19.self_attn.1.q_b_proj.scales": "model-00116-of-00206.safetensors",
+ "model.layers.19.self_attn.1.q_b_proj.weight": "model-00116-of-00206.safetensors",
+ "model.layers.19.self_attn.1.unembed_out.biases": "model-00116-of-00206.safetensors",
+ "model.layers.19.self_attn.1.unembed_out.scales": "model-00116-of-00206.safetensors",
+ "model.layers.19.self_attn.1.unembed_out.weight": "model-00116-of-00206.safetensors",
+ "model.layers.2.input_layernorm.0.weight": "model-00031-of-00206.safetensors",
+ "model.layers.2.input_layernorm.1.weight": "model-00031-of-00206.safetensors",
+ "model.layers.2.mlp.router.classifier.biases": "model-00031-of-00206.safetensors",
+ "model.layers.2.mlp.router.classifier.scales": "model-00031-of-00206.safetensors",
+ "model.layers.2.mlp.router.classifier.weight": "model-00031-of-00206.safetensors",
+ "model.layers.2.mlp.router.e_score_correction_bias": "model-00031-of-00206.safetensors",
+ "model.layers.2.mlp.switch_mlp.down_proj.biases": "model-00031-of-00206.safetensors",
+ "model.layers.2.mlp.switch_mlp.down_proj.scales": "model-00030-of-00206.safetensors",
+ "model.layers.2.mlp.switch_mlp.down_proj.weight": "model-00030-of-00206.safetensors",
+ "model.layers.2.mlp.switch_mlp.gate_proj.biases": "model-00028-of-00206.safetensors",
+ "model.layers.2.mlp.switch_mlp.gate_proj.scales": "model-00027-of-00206.safetensors",
+ "model.layers.2.mlp.switch_mlp.gate_proj.weight": "model-00027-of-00206.safetensors",
+ "model.layers.2.mlp.switch_mlp.up_proj.biases": "model-00029-of-00206.safetensors",
+ "model.layers.2.mlp.switch_mlp.up_proj.scales": "model-00029-of-00206.safetensors",
+ "model.layers.2.mlp.switch_mlp.up_proj.weight": "model-00028-of-00206.safetensors",
+ "model.layers.2.mlps.0.down_proj.biases": "model-00031-of-00206.safetensors",
+ "model.layers.2.mlps.0.down_proj.scales": "model-00031-of-00206.safetensors",
+ "model.layers.2.mlps.0.down_proj.weight": "model-00031-of-00206.safetensors",
+ "model.layers.2.mlps.0.gate_proj.biases": "model-00031-of-00206.safetensors",
+ "model.layers.2.mlps.0.gate_proj.scales": "model-00031-of-00206.safetensors",
+ "model.layers.2.mlps.0.gate_proj.weight": "model-00031-of-00206.safetensors",
+ "model.layers.2.mlps.0.up_proj.biases": "model-00031-of-00206.safetensors",
+ "model.layers.2.mlps.0.up_proj.scales": "model-00031-of-00206.safetensors",
+ "model.layers.2.mlps.0.up_proj.weight": "model-00031-of-00206.safetensors",
+ "model.layers.2.mlps.1.down_proj.biases": "model-00031-of-00206.safetensors",
+ "model.layers.2.mlps.1.down_proj.scales": "model-00031-of-00206.safetensors",
+ "model.layers.2.mlps.1.down_proj.weight": "model-00031-of-00206.safetensors",
+ "model.layers.2.mlps.1.gate_proj.biases": "model-00031-of-00206.safetensors",
+ "model.layers.2.mlps.1.gate_proj.scales": "model-00031-of-00206.safetensors",
+ "model.layers.2.mlps.1.gate_proj.weight": "model-00031-of-00206.safetensors",
+ "model.layers.2.mlps.1.up_proj.biases": "model-00031-of-00206.safetensors",
+ "model.layers.2.mlps.1.up_proj.scales": "model-00031-of-00206.safetensors",
+ "model.layers.2.mlps.1.up_proj.weight": "model-00031-of-00206.safetensors",
+ "model.layers.2.post_attention_layernorm.0.weight": "model-00031-of-00206.safetensors",
+ "model.layers.2.post_attention_layernorm.1.weight": "model-00031-of-00206.safetensors",
+ "model.layers.2.self_attn.0.embed_q.biases": "model-00031-of-00206.safetensors",
+ "model.layers.2.self_attn.0.embed_q.scales": "model-00031-of-00206.safetensors",
+ "model.layers.2.self_attn.0.embed_q.weight": "model-00031-of-00206.safetensors",
+ "model.layers.2.self_attn.0.indexer.k_norm.weight": "model-00031-of-00206.safetensors",
+ "model.layers.2.self_attn.0.indexer.weights_proj.biases": "model-00031-of-00206.safetensors",
+ "model.layers.2.self_attn.0.indexer.weights_proj.scales": "model-00031-of-00206.safetensors",
+ "model.layers.2.self_attn.0.indexer.weights_proj.weight": "model-00031-of-00206.safetensors",
+ "model.layers.2.self_attn.0.indexer.wk.biases": "model-00031-of-00206.safetensors",
+ "model.layers.2.self_attn.0.indexer.wk.scales": "model-00031-of-00206.safetensors",
+ "model.layers.2.self_attn.0.indexer.wk.weight": "model-00031-of-00206.safetensors",
+ "model.layers.2.self_attn.0.indexer.wq_b.biases": "model-00031-of-00206.safetensors",
+ "model.layers.2.self_attn.0.indexer.wq_b.scales": "model-00031-of-00206.safetensors",
+ "model.layers.2.self_attn.0.indexer.wq_b.weight": "model-00031-of-00206.safetensors",
+ "model.layers.2.self_attn.0.kv_a_layernorm.weight": "model-00031-of-00206.safetensors",
+ "model.layers.2.self_attn.0.kv_a_proj_with_mqa.biases": "model-00031-of-00206.safetensors",
+ "model.layers.2.self_attn.0.kv_a_proj_with_mqa.scales": "model-00031-of-00206.safetensors",
+ "model.layers.2.self_attn.0.kv_a_proj_with_mqa.weight": "model-00031-of-00206.safetensors",
+ "model.layers.2.self_attn.0.o_proj.biases": "model-00031-of-00206.safetensors",
+ "model.layers.2.self_attn.0.o_proj.scales": "model-00031-of-00206.safetensors",
+ "model.layers.2.self_attn.0.o_proj.weight": "model-00031-of-00206.safetensors",
+ "model.layers.2.self_attn.0.q_a_layernorm.weight": "model-00031-of-00206.safetensors",
+ "model.layers.2.self_attn.0.q_a_proj.biases": "model-00031-of-00206.safetensors",
+ "model.layers.2.self_attn.0.q_a_proj.scales": "model-00031-of-00206.safetensors",
+ "model.layers.2.self_attn.0.q_a_proj.weight": "model-00031-of-00206.safetensors",
+ "model.layers.2.self_attn.0.q_b_proj.biases": "model-00031-of-00206.safetensors",
+ "model.layers.2.self_attn.0.q_b_proj.scales": "model-00031-of-00206.safetensors",
+ "model.layers.2.self_attn.0.q_b_proj.weight": "model-00031-of-00206.safetensors",
+ "model.layers.2.self_attn.0.unembed_out.biases": "model-00031-of-00206.safetensors",
+ "model.layers.2.self_attn.0.unembed_out.scales": "model-00031-of-00206.safetensors",
+ "model.layers.2.self_attn.0.unembed_out.weight": "model-00031-of-00206.safetensors",
+ "model.layers.2.self_attn.1.embed_q.biases": "model-00031-of-00206.safetensors",
+ "model.layers.2.self_attn.1.embed_q.scales": "model-00031-of-00206.safetensors",
+ "model.layers.2.self_attn.1.embed_q.weight": "model-00031-of-00206.safetensors",
+ "model.layers.2.self_attn.1.kv_a_layernorm.weight": "model-00031-of-00206.safetensors",
+ "model.layers.2.self_attn.1.kv_a_proj_with_mqa.biases": "model-00031-of-00206.safetensors",
+ "model.layers.2.self_attn.1.kv_a_proj_with_mqa.scales": "model-00031-of-00206.safetensors",
+ "model.layers.2.self_attn.1.kv_a_proj_with_mqa.weight": "model-00031-of-00206.safetensors",
+ "model.layers.2.self_attn.1.o_proj.biases": "model-00031-of-00206.safetensors",
+ "model.layers.2.self_attn.1.o_proj.scales": "model-00031-of-00206.safetensors",
+ "model.layers.2.self_attn.1.o_proj.weight": "model-00031-of-00206.safetensors",
+ "model.layers.2.self_attn.1.q_a_layernorm.weight": "model-00031-of-00206.safetensors",
+ "model.layers.2.self_attn.1.q_a_proj.biases": "model-00031-of-00206.safetensors",
+ "model.layers.2.self_attn.1.q_a_proj.scales": "model-00031-of-00206.safetensors",
+ "model.layers.2.self_attn.1.q_a_proj.weight": "model-00031-of-00206.safetensors",
+ "model.layers.2.self_attn.1.q_b_proj.biases": "model-00031-of-00206.safetensors",
+ "model.layers.2.self_attn.1.q_b_proj.scales": "model-00031-of-00206.safetensors",
+ "model.layers.2.self_attn.1.q_b_proj.weight": "model-00031-of-00206.safetensors",
+ "model.layers.2.self_attn.1.unembed_out.biases": "model-00031-of-00206.safetensors",
+ "model.layers.2.self_attn.1.unembed_out.scales": "model-00031-of-00206.safetensors",
+ "model.layers.2.self_attn.1.unembed_out.weight": "model-00031-of-00206.safetensors",
+ "model.layers.20.input_layernorm.0.weight": "model-00121-of-00206.safetensors",
+ "model.layers.20.input_layernorm.1.weight": "model-00121-of-00206.safetensors",
+ "model.layers.20.mlp.router.classifier.biases": "model-00121-of-00206.safetensors",
+ "model.layers.20.mlp.router.classifier.scales": "model-00121-of-00206.safetensors",
+ "model.layers.20.mlp.router.classifier.weight": "model-00121-of-00206.safetensors",
+ "model.layers.20.mlp.router.e_score_correction_bias": "model-00121-of-00206.safetensors",
+ "model.layers.20.mlp.switch_mlp.down_proj.biases": "model-00121-of-00206.safetensors",
+ "model.layers.20.mlp.switch_mlp.down_proj.scales": "model-00120-of-00206.safetensors",
+ "model.layers.20.mlp.switch_mlp.down_proj.weight": "model-00120-of-00206.safetensors",
+ "model.layers.20.mlp.switch_mlp.gate_proj.biases": "model-00118-of-00206.safetensors",
+ "model.layers.20.mlp.switch_mlp.gate_proj.scales": "model-00117-of-00206.safetensors",
+ "model.layers.20.mlp.switch_mlp.gate_proj.weight": "model-00117-of-00206.safetensors",
+ "model.layers.20.mlp.switch_mlp.up_proj.biases": "model-00119-of-00206.safetensors",
+ "model.layers.20.mlp.switch_mlp.up_proj.scales": "model-00119-of-00206.safetensors",
+ "model.layers.20.mlp.switch_mlp.up_proj.weight": "model-00118-of-00206.safetensors",
+ "model.layers.20.mlps.0.down_proj.biases": "model-00121-of-00206.safetensors",
+ "model.layers.20.mlps.0.down_proj.scales": "model-00121-of-00206.safetensors",
+ "model.layers.20.mlps.0.down_proj.weight": "model-00121-of-00206.safetensors",
+ "model.layers.20.mlps.0.gate_proj.biases": "model-00121-of-00206.safetensors",
+ "model.layers.20.mlps.0.gate_proj.scales": "model-00121-of-00206.safetensors",
+ "model.layers.20.mlps.0.gate_proj.weight": "model-00121-of-00206.safetensors",
+ "model.layers.20.mlps.0.up_proj.biases": "model-00121-of-00206.safetensors",
+ "model.layers.20.mlps.0.up_proj.scales": "model-00121-of-00206.safetensors",
+ "model.layers.20.mlps.0.up_proj.weight": "model-00121-of-00206.safetensors",
+ "model.layers.20.mlps.1.down_proj.biases": "model-00121-of-00206.safetensors",
+ "model.layers.20.mlps.1.down_proj.scales": "model-00121-of-00206.safetensors",
+ "model.layers.20.mlps.1.down_proj.weight": "model-00121-of-00206.safetensors",
+ "model.layers.20.mlps.1.gate_proj.biases": "model-00121-of-00206.safetensors",
+ "model.layers.20.mlps.1.gate_proj.scales": "model-00121-of-00206.safetensors",
+ "model.layers.20.mlps.1.gate_proj.weight": "model-00121-of-00206.safetensors",
+ "model.layers.20.mlps.1.up_proj.biases": "model-00121-of-00206.safetensors",
+ "model.layers.20.mlps.1.up_proj.scales": "model-00121-of-00206.safetensors",
+ "model.layers.20.mlps.1.up_proj.weight": "model-00121-of-00206.safetensors",
+ "model.layers.20.post_attention_layernorm.0.weight": "model-00121-of-00206.safetensors",
+ "model.layers.20.post_attention_layernorm.1.weight": "model-00121-of-00206.safetensors",
+ "model.layers.20.self_attn.0.embed_q.biases": "model-00121-of-00206.safetensors",
+ "model.layers.20.self_attn.0.embed_q.scales": "model-00121-of-00206.safetensors",
+ "model.layers.20.self_attn.0.embed_q.weight": "model-00121-of-00206.safetensors",
+ "model.layers.20.self_attn.0.indexer.k_norm.weight": "model-00121-of-00206.safetensors",
+ "model.layers.20.self_attn.0.indexer.weights_proj.biases": "model-00121-of-00206.safetensors",
+ "model.layers.20.self_attn.0.indexer.weights_proj.scales": "model-00121-of-00206.safetensors",
+ "model.layers.20.self_attn.0.indexer.weights_proj.weight": "model-00121-of-00206.safetensors",
+ "model.layers.20.self_attn.0.indexer.wk.biases": "model-00121-of-00206.safetensors",
+ "model.layers.20.self_attn.0.indexer.wk.scales": "model-00121-of-00206.safetensors",
+ "model.layers.20.self_attn.0.indexer.wk.weight": "model-00121-of-00206.safetensors",
+ "model.layers.20.self_attn.0.indexer.wq_b.biases": "model-00121-of-00206.safetensors",
+ "model.layers.20.self_attn.0.indexer.wq_b.scales": "model-00121-of-00206.safetensors",
+ "model.layers.20.self_attn.0.indexer.wq_b.weight": "model-00121-of-00206.safetensors",
+ "model.layers.20.self_attn.0.kv_a_layernorm.weight": "model-00121-of-00206.safetensors",
+ "model.layers.20.self_attn.0.kv_a_proj_with_mqa.biases": "model-00121-of-00206.safetensors",
+ "model.layers.20.self_attn.0.kv_a_proj_with_mqa.scales": "model-00121-of-00206.safetensors",
+ "model.layers.20.self_attn.0.kv_a_proj_with_mqa.weight": "model-00121-of-00206.safetensors",
+ "model.layers.20.self_attn.0.o_proj.biases": "model-00121-of-00206.safetensors",
+ "model.layers.20.self_attn.0.o_proj.scales": "model-00121-of-00206.safetensors",
+ "model.layers.20.self_attn.0.o_proj.weight": "model-00121-of-00206.safetensors",
+ "model.layers.20.self_attn.0.q_a_layernorm.weight": "model-00121-of-00206.safetensors",
+ "model.layers.20.self_attn.0.q_a_proj.biases": "model-00121-of-00206.safetensors",
+ "model.layers.20.self_attn.0.q_a_proj.scales": "model-00121-of-00206.safetensors",
+ "model.layers.20.self_attn.0.q_a_proj.weight": "model-00121-of-00206.safetensors",
+ "model.layers.20.self_attn.0.q_b_proj.biases": "model-00121-of-00206.safetensors",
+ "model.layers.20.self_attn.0.q_b_proj.scales": "model-00121-of-00206.safetensors",
+ "model.layers.20.self_attn.0.q_b_proj.weight": "model-00121-of-00206.safetensors",
+ "model.layers.20.self_attn.0.unembed_out.biases": "model-00121-of-00206.safetensors",
+ "model.layers.20.self_attn.0.unembed_out.scales": "model-00121-of-00206.safetensors",
+ "model.layers.20.self_attn.0.unembed_out.weight": "model-00121-of-00206.safetensors",
+ "model.layers.20.self_attn.1.embed_q.biases": "model-00121-of-00206.safetensors",
+ "model.layers.20.self_attn.1.embed_q.scales": "model-00121-of-00206.safetensors",
+ "model.layers.20.self_attn.1.embed_q.weight": "model-00121-of-00206.safetensors",
+ "model.layers.20.self_attn.1.kv_a_layernorm.weight": "model-00121-of-00206.safetensors",
+ "model.layers.20.self_attn.1.kv_a_proj_with_mqa.biases": "model-00121-of-00206.safetensors",
+ "model.layers.20.self_attn.1.kv_a_proj_with_mqa.scales": "model-00121-of-00206.safetensors",
+ "model.layers.20.self_attn.1.kv_a_proj_with_mqa.weight": "model-00121-of-00206.safetensors",
+ "model.layers.20.self_attn.1.o_proj.biases": "model-00121-of-00206.safetensors",
+ "model.layers.20.self_attn.1.o_proj.scales": "model-00121-of-00206.safetensors",
+ "model.layers.20.self_attn.1.o_proj.weight": "model-00121-of-00206.safetensors",
+ "model.layers.20.self_attn.1.q_a_layernorm.weight": "model-00121-of-00206.safetensors",
+ "model.layers.20.self_attn.1.q_a_proj.biases": "model-00121-of-00206.safetensors",
+ "model.layers.20.self_attn.1.q_a_proj.scales": "model-00121-of-00206.safetensors",
+ "model.layers.20.self_attn.1.q_a_proj.weight": "model-00121-of-00206.safetensors",
+ "model.layers.20.self_attn.1.q_b_proj.biases": "model-00121-of-00206.safetensors",
+ "model.layers.20.self_attn.1.q_b_proj.scales": "model-00121-of-00206.safetensors",
+ "model.layers.20.self_attn.1.q_b_proj.weight": "model-00121-of-00206.safetensors",
+ "model.layers.20.self_attn.1.unembed_out.biases": "model-00121-of-00206.safetensors",
+ "model.layers.20.self_attn.1.unembed_out.scales": "model-00121-of-00206.safetensors",
+ "model.layers.20.self_attn.1.unembed_out.weight": "model-00121-of-00206.safetensors",
+ "model.layers.21.input_layernorm.0.weight": "model-00126-of-00206.safetensors",
+ "model.layers.21.input_layernorm.1.weight": "model-00126-of-00206.safetensors",
+ "model.layers.21.mlp.router.classifier.biases": "model-00126-of-00206.safetensors",
+ "model.layers.21.mlp.router.classifier.scales": "model-00126-of-00206.safetensors",
+ "model.layers.21.mlp.router.classifier.weight": "model-00126-of-00206.safetensors",
+ "model.layers.21.mlp.router.e_score_correction_bias": "model-00126-of-00206.safetensors",
+ "model.layers.21.mlp.switch_mlp.down_proj.biases": "model-00126-of-00206.safetensors",
+ "model.layers.21.mlp.switch_mlp.down_proj.scales": "model-00125-of-00206.safetensors",
+ "model.layers.21.mlp.switch_mlp.down_proj.weight": "model-00125-of-00206.safetensors",
+ "model.layers.21.mlp.switch_mlp.gate_proj.biases": "model-00123-of-00206.safetensors",
+ "model.layers.21.mlp.switch_mlp.gate_proj.scales": "model-00122-of-00206.safetensors",
+ "model.layers.21.mlp.switch_mlp.gate_proj.weight": "model-00122-of-00206.safetensors",
+ "model.layers.21.mlp.switch_mlp.up_proj.biases": "model-00124-of-00206.safetensors",
+ "model.layers.21.mlp.switch_mlp.up_proj.scales": "model-00124-of-00206.safetensors",
+ "model.layers.21.mlp.switch_mlp.up_proj.weight": "model-00123-of-00206.safetensors",
+ "model.layers.21.mlps.0.down_proj.biases": "model-00126-of-00206.safetensors",
+ "model.layers.21.mlps.0.down_proj.scales": "model-00126-of-00206.safetensors",
+ "model.layers.21.mlps.0.down_proj.weight": "model-00126-of-00206.safetensors",
+ "model.layers.21.mlps.0.gate_proj.biases": "model-00126-of-00206.safetensors",
+ "model.layers.21.mlps.0.gate_proj.scales": "model-00126-of-00206.safetensors",
+ "model.layers.21.mlps.0.gate_proj.weight": "model-00126-of-00206.safetensors",
+ "model.layers.21.mlps.0.up_proj.biases": "model-00126-of-00206.safetensors",
+ "model.layers.21.mlps.0.up_proj.scales": "model-00126-of-00206.safetensors",
+ "model.layers.21.mlps.0.up_proj.weight": "model-00126-of-00206.safetensors",
+ "model.layers.21.mlps.1.down_proj.biases": "model-00126-of-00206.safetensors",
+ "model.layers.21.mlps.1.down_proj.scales": "model-00126-of-00206.safetensors",
+ "model.layers.21.mlps.1.down_proj.weight": "model-00126-of-00206.safetensors",
+ "model.layers.21.mlps.1.gate_proj.biases": "model-00126-of-00206.safetensors",
+ "model.layers.21.mlps.1.gate_proj.scales": "model-00126-of-00206.safetensors",
+ "model.layers.21.mlps.1.gate_proj.weight": "model-00126-of-00206.safetensors",
+ "model.layers.21.mlps.1.up_proj.biases": "model-00126-of-00206.safetensors",
+ "model.layers.21.mlps.1.up_proj.scales": "model-00126-of-00206.safetensors",
+ "model.layers.21.mlps.1.up_proj.weight": "model-00126-of-00206.safetensors",
+ "model.layers.21.post_attention_layernorm.0.weight": "model-00126-of-00206.safetensors",
+ "model.layers.21.post_attention_layernorm.1.weight": "model-00126-of-00206.safetensors",
+ "model.layers.21.self_attn.0.embed_q.biases": "model-00126-of-00206.safetensors",
+ "model.layers.21.self_attn.0.embed_q.scales": "model-00126-of-00206.safetensors",
+ "model.layers.21.self_attn.0.embed_q.weight": "model-00126-of-00206.safetensors",
+ "model.layers.21.self_attn.0.indexer.k_norm.weight": "model-00126-of-00206.safetensors",
+ "model.layers.21.self_attn.0.indexer.weights_proj.biases": "model-00126-of-00206.safetensors",
+ "model.layers.21.self_attn.0.indexer.weights_proj.scales": "model-00126-of-00206.safetensors",
+ "model.layers.21.self_attn.0.indexer.weights_proj.weight": "model-00126-of-00206.safetensors",
+ "model.layers.21.self_attn.0.indexer.wk.biases": "model-00126-of-00206.safetensors",
+ "model.layers.21.self_attn.0.indexer.wk.scales": "model-00126-of-00206.safetensors",
+ "model.layers.21.self_attn.0.indexer.wk.weight": "model-00126-of-00206.safetensors",
+ "model.layers.21.self_attn.0.indexer.wq_b.biases": "model-00126-of-00206.safetensors",
+ "model.layers.21.self_attn.0.indexer.wq_b.scales": "model-00126-of-00206.safetensors",
+ "model.layers.21.self_attn.0.indexer.wq_b.weight": "model-00126-of-00206.safetensors",
+ "model.layers.21.self_attn.0.kv_a_layernorm.weight": "model-00126-of-00206.safetensors",
+ "model.layers.21.self_attn.0.kv_a_proj_with_mqa.biases": "model-00126-of-00206.safetensors",
+ "model.layers.21.self_attn.0.kv_a_proj_with_mqa.scales": "model-00126-of-00206.safetensors",
+ "model.layers.21.self_attn.0.kv_a_proj_with_mqa.weight": "model-00126-of-00206.safetensors",
+ "model.layers.21.self_attn.0.o_proj.biases": "model-00126-of-00206.safetensors",
+ "model.layers.21.self_attn.0.o_proj.scales": "model-00126-of-00206.safetensors",
+ "model.layers.21.self_attn.0.o_proj.weight": "model-00126-of-00206.safetensors",
+ "model.layers.21.self_attn.0.q_a_layernorm.weight": "model-00126-of-00206.safetensors",
+ "model.layers.21.self_attn.0.q_a_proj.biases": "model-00126-of-00206.safetensors",
+ "model.layers.21.self_attn.0.q_a_proj.scales": "model-00126-of-00206.safetensors",
+ "model.layers.21.self_attn.0.q_a_proj.weight": "model-00126-of-00206.safetensors",
+ "model.layers.21.self_attn.0.q_b_proj.biases": "model-00126-of-00206.safetensors",
+ "model.layers.21.self_attn.0.q_b_proj.scales": "model-00126-of-00206.safetensors",
+ "model.layers.21.self_attn.0.q_b_proj.weight": "model-00126-of-00206.safetensors",
+ "model.layers.21.self_attn.0.unembed_out.biases": "model-00126-of-00206.safetensors",
+ "model.layers.21.self_attn.0.unembed_out.scales": "model-00126-of-00206.safetensors",
+ "model.layers.21.self_attn.0.unembed_out.weight": "model-00126-of-00206.safetensors",
+ "model.layers.21.self_attn.1.embed_q.biases": "model-00126-of-00206.safetensors",
+ "model.layers.21.self_attn.1.embed_q.scales": "model-00126-of-00206.safetensors",
+ "model.layers.21.self_attn.1.embed_q.weight": "model-00126-of-00206.safetensors",
+ "model.layers.21.self_attn.1.kv_a_layernorm.weight": "model-00126-of-00206.safetensors",
+ "model.layers.21.self_attn.1.kv_a_proj_with_mqa.biases": "model-00126-of-00206.safetensors",
+ "model.layers.21.self_attn.1.kv_a_proj_with_mqa.scales": "model-00126-of-00206.safetensors",
+ "model.layers.21.self_attn.1.kv_a_proj_with_mqa.weight": "model-00126-of-00206.safetensors",
+ "model.layers.21.self_attn.1.o_proj.biases": "model-00126-of-00206.safetensors",
+ "model.layers.21.self_attn.1.o_proj.scales": "model-00126-of-00206.safetensors",
+ "model.layers.21.self_attn.1.o_proj.weight": "model-00126-of-00206.safetensors",
+ "model.layers.21.self_attn.1.q_a_layernorm.weight": "model-00126-of-00206.safetensors",
+ "model.layers.21.self_attn.1.q_a_proj.biases": "model-00126-of-00206.safetensors",
+ "model.layers.21.self_attn.1.q_a_proj.scales": "model-00126-of-00206.safetensors",
+ "model.layers.21.self_attn.1.q_a_proj.weight": "model-00126-of-00206.safetensors",
+ "model.layers.21.self_attn.1.q_b_proj.biases": "model-00126-of-00206.safetensors",
+ "model.layers.21.self_attn.1.q_b_proj.scales": "model-00126-of-00206.safetensors",
+ "model.layers.21.self_attn.1.q_b_proj.weight": "model-00126-of-00206.safetensors",
+ "model.layers.21.self_attn.1.unembed_out.biases": "model-00126-of-00206.safetensors",
+ "model.layers.21.self_attn.1.unembed_out.scales": "model-00126-of-00206.safetensors",
+ "model.layers.21.self_attn.1.unembed_out.weight": "model-00126-of-00206.safetensors",
+ "model.layers.22.input_layernorm.0.weight": "model-00131-of-00206.safetensors",
+ "model.layers.22.input_layernorm.1.weight": "model-00131-of-00206.safetensors",
+ "model.layers.22.mlp.router.classifier.biases": "model-00131-of-00206.safetensors",
+ "model.layers.22.mlp.router.classifier.scales": "model-00131-of-00206.safetensors",
+ "model.layers.22.mlp.router.classifier.weight": "model-00131-of-00206.safetensors",
+ "model.layers.22.mlp.router.e_score_correction_bias": "model-00131-of-00206.safetensors",
+ "model.layers.22.mlp.switch_mlp.down_proj.biases": "model-00131-of-00206.safetensors",
+ "model.layers.22.mlp.switch_mlp.down_proj.scales": "model-00130-of-00206.safetensors",
+ "model.layers.22.mlp.switch_mlp.down_proj.weight": "model-00130-of-00206.safetensors",
+ "model.layers.22.mlp.switch_mlp.gate_proj.biases": "model-00128-of-00206.safetensors",
+ "model.layers.22.mlp.switch_mlp.gate_proj.scales": "model-00127-of-00206.safetensors",
+ "model.layers.22.mlp.switch_mlp.gate_proj.weight": "model-00127-of-00206.safetensors",
+ "model.layers.22.mlp.switch_mlp.up_proj.biases": "model-00129-of-00206.safetensors",
+ "model.layers.22.mlp.switch_mlp.up_proj.scales": "model-00129-of-00206.safetensors",
+ "model.layers.22.mlp.switch_mlp.up_proj.weight": "model-00128-of-00206.safetensors",
+ "model.layers.22.mlps.0.down_proj.biases": "model-00131-of-00206.safetensors",
+ "model.layers.22.mlps.0.down_proj.scales": "model-00131-of-00206.safetensors",
+ "model.layers.22.mlps.0.down_proj.weight": "model-00131-of-00206.safetensors",
+ "model.layers.22.mlps.0.gate_proj.biases": "model-00131-of-00206.safetensors",
+ "model.layers.22.mlps.0.gate_proj.scales": "model-00131-of-00206.safetensors",
+ "model.layers.22.mlps.0.gate_proj.weight": "model-00131-of-00206.safetensors",
+ "model.layers.22.mlps.0.up_proj.biases": "model-00131-of-00206.safetensors",
+ "model.layers.22.mlps.0.up_proj.scales": "model-00131-of-00206.safetensors",
+ "model.layers.22.mlps.0.up_proj.weight": "model-00131-of-00206.safetensors",
+ "model.layers.22.mlps.1.down_proj.biases": "model-00131-of-00206.safetensors",
+ "model.layers.22.mlps.1.down_proj.scales": "model-00131-of-00206.safetensors",
+ "model.layers.22.mlps.1.down_proj.weight": "model-00131-of-00206.safetensors",
+ "model.layers.22.mlps.1.gate_proj.biases": "model-00131-of-00206.safetensors",
+ "model.layers.22.mlps.1.gate_proj.scales": "model-00131-of-00206.safetensors",
+ "model.layers.22.mlps.1.gate_proj.weight": "model-00131-of-00206.safetensors",
+ "model.layers.22.mlps.1.up_proj.biases": "model-00131-of-00206.safetensors",
+ "model.layers.22.mlps.1.up_proj.scales": "model-00131-of-00206.safetensors",
+ "model.layers.22.mlps.1.up_proj.weight": "model-00131-of-00206.safetensors",
+ "model.layers.22.post_attention_layernorm.0.weight": "model-00131-of-00206.safetensors",
+ "model.layers.22.post_attention_layernorm.1.weight": "model-00131-of-00206.safetensors",
+ "model.layers.22.self_attn.0.embed_q.biases": "model-00131-of-00206.safetensors",
+ "model.layers.22.self_attn.0.embed_q.scales": "model-00131-of-00206.safetensors",
+ "model.layers.22.self_attn.0.embed_q.weight": "model-00131-of-00206.safetensors",
+ "model.layers.22.self_attn.0.indexer.k_norm.weight": "model-00131-of-00206.safetensors",
+ "model.layers.22.self_attn.0.indexer.weights_proj.biases": "model-00131-of-00206.safetensors",
+ "model.layers.22.self_attn.0.indexer.weights_proj.scales": "model-00131-of-00206.safetensors",
+ "model.layers.22.self_attn.0.indexer.weights_proj.weight": "model-00131-of-00206.safetensors",
+ "model.layers.22.self_attn.0.indexer.wk.biases": "model-00131-of-00206.safetensors",
+ "model.layers.22.self_attn.0.indexer.wk.scales": "model-00131-of-00206.safetensors",
+ "model.layers.22.self_attn.0.indexer.wk.weight": "model-00131-of-00206.safetensors",
+ "model.layers.22.self_attn.0.indexer.wq_b.biases": "model-00131-of-00206.safetensors",
+ "model.layers.22.self_attn.0.indexer.wq_b.scales": "model-00131-of-00206.safetensors",
+ "model.layers.22.self_attn.0.indexer.wq_b.weight": "model-00131-of-00206.safetensors",
+ "model.layers.22.self_attn.0.kv_a_layernorm.weight": "model-00131-of-00206.safetensors",
+ "model.layers.22.self_attn.0.kv_a_proj_with_mqa.biases": "model-00131-of-00206.safetensors",
+ "model.layers.22.self_attn.0.kv_a_proj_with_mqa.scales": "model-00131-of-00206.safetensors",
+ "model.layers.22.self_attn.0.kv_a_proj_with_mqa.weight": "model-00131-of-00206.safetensors",
+ "model.layers.22.self_attn.0.o_proj.biases": "model-00131-of-00206.safetensors",
+ "model.layers.22.self_attn.0.o_proj.scales": "model-00131-of-00206.safetensors",
+ "model.layers.22.self_attn.0.o_proj.weight": "model-00131-of-00206.safetensors",
+ "model.layers.22.self_attn.0.q_a_layernorm.weight": "model-00131-of-00206.safetensors",
+ "model.layers.22.self_attn.0.q_a_proj.biases": "model-00131-of-00206.safetensors",
+ "model.layers.22.self_attn.0.q_a_proj.scales": "model-00131-of-00206.safetensors",
+ "model.layers.22.self_attn.0.q_a_proj.weight": "model-00131-of-00206.safetensors",
+ "model.layers.22.self_attn.0.q_b_proj.biases": "model-00131-of-00206.safetensors",
+ "model.layers.22.self_attn.0.q_b_proj.scales": "model-00131-of-00206.safetensors",
+ "model.layers.22.self_attn.0.q_b_proj.weight": "model-00131-of-00206.safetensors",
+ "model.layers.22.self_attn.0.unembed_out.biases": "model-00131-of-00206.safetensors",
+ "model.layers.22.self_attn.0.unembed_out.scales": "model-00131-of-00206.safetensors",
+ "model.layers.22.self_attn.0.unembed_out.weight": "model-00131-of-00206.safetensors",
+ "model.layers.22.self_attn.1.embed_q.biases": "model-00131-of-00206.safetensors",
+ "model.layers.22.self_attn.1.embed_q.scales": "model-00131-of-00206.safetensors",
+ "model.layers.22.self_attn.1.embed_q.weight": "model-00131-of-00206.safetensors",
+ "model.layers.22.self_attn.1.kv_a_layernorm.weight": "model-00131-of-00206.safetensors",
+ "model.layers.22.self_attn.1.kv_a_proj_with_mqa.biases": "model-00131-of-00206.safetensors",
+ "model.layers.22.self_attn.1.kv_a_proj_with_mqa.scales": "model-00131-of-00206.safetensors",
+ "model.layers.22.self_attn.1.kv_a_proj_with_mqa.weight": "model-00131-of-00206.safetensors",
+ "model.layers.22.self_attn.1.o_proj.biases": "model-00131-of-00206.safetensors",
+ "model.layers.22.self_attn.1.o_proj.scales": "model-00131-of-00206.safetensors",
+ "model.layers.22.self_attn.1.o_proj.weight": "model-00131-of-00206.safetensors",
+ "model.layers.22.self_attn.1.q_a_layernorm.weight": "model-00131-of-00206.safetensors",
+ "model.layers.22.self_attn.1.q_a_proj.biases": "model-00131-of-00206.safetensors",
+ "model.layers.22.self_attn.1.q_a_proj.scales": "model-00131-of-00206.safetensors",
+ "model.layers.22.self_attn.1.q_a_proj.weight": "model-00131-of-00206.safetensors",
+ "model.layers.22.self_attn.1.q_b_proj.biases": "model-00131-of-00206.safetensors",
+ "model.layers.22.self_attn.1.q_b_proj.scales": "model-00131-of-00206.safetensors",
+ "model.layers.22.self_attn.1.q_b_proj.weight": "model-00131-of-00206.safetensors",
+ "model.layers.22.self_attn.1.unembed_out.biases": "model-00131-of-00206.safetensors",
+ "model.layers.22.self_attn.1.unembed_out.scales": "model-00131-of-00206.safetensors",
+ "model.layers.22.self_attn.1.unembed_out.weight": "model-00131-of-00206.safetensors",
+ "model.layers.23.input_layernorm.0.weight": "model-00136-of-00206.safetensors",
+ "model.layers.23.input_layernorm.1.weight": "model-00136-of-00206.safetensors",
+ "model.layers.23.mlp.router.classifier.biases": "model-00136-of-00206.safetensors",
+ "model.layers.23.mlp.router.classifier.scales": "model-00136-of-00206.safetensors",
+ "model.layers.23.mlp.router.classifier.weight": "model-00136-of-00206.safetensors",
+ "model.layers.23.mlp.router.e_score_correction_bias": "model-00136-of-00206.safetensors",
+ "model.layers.23.mlp.switch_mlp.down_proj.biases": "model-00136-of-00206.safetensors",
+ "model.layers.23.mlp.switch_mlp.down_proj.scales": "model-00135-of-00206.safetensors",
+ "model.layers.23.mlp.switch_mlp.down_proj.weight": "model-00135-of-00206.safetensors",
+ "model.layers.23.mlp.switch_mlp.gate_proj.biases": "model-00133-of-00206.safetensors",
+ "model.layers.23.mlp.switch_mlp.gate_proj.scales": "model-00132-of-00206.safetensors",
+ "model.layers.23.mlp.switch_mlp.gate_proj.weight": "model-00132-of-00206.safetensors",
+ "model.layers.23.mlp.switch_mlp.up_proj.biases": "model-00134-of-00206.safetensors",
+ "model.layers.23.mlp.switch_mlp.up_proj.scales": "model-00134-of-00206.safetensors",
+ "model.layers.23.mlp.switch_mlp.up_proj.weight": "model-00133-of-00206.safetensors",
+ "model.layers.23.mlps.0.down_proj.biases": "model-00136-of-00206.safetensors",
+ "model.layers.23.mlps.0.down_proj.scales": "model-00136-of-00206.safetensors",
+ "model.layers.23.mlps.0.down_proj.weight": "model-00136-of-00206.safetensors",
+ "model.layers.23.mlps.0.gate_proj.biases": "model-00136-of-00206.safetensors",
+ "model.layers.23.mlps.0.gate_proj.scales": "model-00136-of-00206.safetensors",
+ "model.layers.23.mlps.0.gate_proj.weight": "model-00136-of-00206.safetensors",
+ "model.layers.23.mlps.0.up_proj.biases": "model-00136-of-00206.safetensors",
+ "model.layers.23.mlps.0.up_proj.scales": "model-00136-of-00206.safetensors",
+ "model.layers.23.mlps.0.up_proj.weight": "model-00136-of-00206.safetensors",
+ "model.layers.23.mlps.1.down_proj.biases": "model-00136-of-00206.safetensors",
+ "model.layers.23.mlps.1.down_proj.scales": "model-00136-of-00206.safetensors",
+ "model.layers.23.mlps.1.down_proj.weight": "model-00136-of-00206.safetensors",
+ "model.layers.23.mlps.1.gate_proj.biases": "model-00136-of-00206.safetensors",
+ "model.layers.23.mlps.1.gate_proj.scales": "model-00136-of-00206.safetensors",
+ "model.layers.23.mlps.1.gate_proj.weight": "model-00136-of-00206.safetensors",
+ "model.layers.23.mlps.1.up_proj.biases": "model-00136-of-00206.safetensors",
+ "model.layers.23.mlps.1.up_proj.scales": "model-00136-of-00206.safetensors",
+ "model.layers.23.mlps.1.up_proj.weight": "model-00136-of-00206.safetensors",
+ "model.layers.23.post_attention_layernorm.0.weight": "model-00136-of-00206.safetensors",
+ "model.layers.23.post_attention_layernorm.1.weight": "model-00136-of-00206.safetensors",
+ "model.layers.23.self_attn.0.embed_q.biases": "model-00136-of-00206.safetensors",
+ "model.layers.23.self_attn.0.embed_q.scales": "model-00136-of-00206.safetensors",
+ "model.layers.23.self_attn.0.embed_q.weight": "model-00136-of-00206.safetensors",
+ "model.layers.23.self_attn.0.indexer.k_norm.weight": "model-00136-of-00206.safetensors",
+ "model.layers.23.self_attn.0.indexer.weights_proj.biases": "model-00136-of-00206.safetensors",
+ "model.layers.23.self_attn.0.indexer.weights_proj.scales": "model-00136-of-00206.safetensors",
+ "model.layers.23.self_attn.0.indexer.weights_proj.weight": "model-00136-of-00206.safetensors",
+ "model.layers.23.self_attn.0.indexer.wk.biases": "model-00136-of-00206.safetensors",
+ "model.layers.23.self_attn.0.indexer.wk.scales": "model-00136-of-00206.safetensors",
+ "model.layers.23.self_attn.0.indexer.wk.weight": "model-00136-of-00206.safetensors",
+ "model.layers.23.self_attn.0.indexer.wq_b.biases": "model-00136-of-00206.safetensors",
+ "model.layers.23.self_attn.0.indexer.wq_b.scales": "model-00136-of-00206.safetensors",
+ "model.layers.23.self_attn.0.indexer.wq_b.weight": "model-00136-of-00206.safetensors",
+ "model.layers.23.self_attn.0.kv_a_layernorm.weight": "model-00136-of-00206.safetensors",
+ "model.layers.23.self_attn.0.kv_a_proj_with_mqa.biases": "model-00136-of-00206.safetensors",
+ "model.layers.23.self_attn.0.kv_a_proj_with_mqa.scales": "model-00136-of-00206.safetensors",
+ "model.layers.23.self_attn.0.kv_a_proj_with_mqa.weight": "model-00136-of-00206.safetensors",
+ "model.layers.23.self_attn.0.o_proj.biases": "model-00136-of-00206.safetensors",
+ "model.layers.23.self_attn.0.o_proj.scales": "model-00136-of-00206.safetensors",
+ "model.layers.23.self_attn.0.o_proj.weight": "model-00136-of-00206.safetensors",
+ "model.layers.23.self_attn.0.q_a_layernorm.weight": "model-00136-of-00206.safetensors",
+ "model.layers.23.self_attn.0.q_a_proj.biases": "model-00136-of-00206.safetensors",
+ "model.layers.23.self_attn.0.q_a_proj.scales": "model-00136-of-00206.safetensors",
+ "model.layers.23.self_attn.0.q_a_proj.weight": "model-00136-of-00206.safetensors",
+ "model.layers.23.self_attn.0.q_b_proj.biases": "model-00136-of-00206.safetensors",
+ "model.layers.23.self_attn.0.q_b_proj.scales": "model-00136-of-00206.safetensors",
+ "model.layers.23.self_attn.0.q_b_proj.weight": "model-00136-of-00206.safetensors",
+ "model.layers.23.self_attn.0.unembed_out.biases": "model-00136-of-00206.safetensors",
+ "model.layers.23.self_attn.0.unembed_out.scales": "model-00136-of-00206.safetensors",
+ "model.layers.23.self_attn.0.unembed_out.weight": "model-00136-of-00206.safetensors",
+ "model.layers.23.self_attn.1.embed_q.biases": "model-00136-of-00206.safetensors",
+ "model.layers.23.self_attn.1.embed_q.scales": "model-00136-of-00206.safetensors",
+ "model.layers.23.self_attn.1.embed_q.weight": "model-00136-of-00206.safetensors",
+ "model.layers.23.self_attn.1.kv_a_layernorm.weight": "model-00136-of-00206.safetensors",
+ "model.layers.23.self_attn.1.kv_a_proj_with_mqa.biases": "model-00136-of-00206.safetensors",
+ "model.layers.23.self_attn.1.kv_a_proj_with_mqa.scales": "model-00136-of-00206.safetensors",
+ "model.layers.23.self_attn.1.kv_a_proj_with_mqa.weight": "model-00136-of-00206.safetensors",
+ "model.layers.23.self_attn.1.o_proj.biases": "model-00136-of-00206.safetensors",
+ "model.layers.23.self_attn.1.o_proj.scales": "model-00136-of-00206.safetensors",
+ "model.layers.23.self_attn.1.o_proj.weight": "model-00136-of-00206.safetensors",
+ "model.layers.23.self_attn.1.q_a_layernorm.weight": "model-00136-of-00206.safetensors",
+ "model.layers.23.self_attn.1.q_a_proj.biases": "model-00136-of-00206.safetensors",
+ "model.layers.23.self_attn.1.q_a_proj.scales": "model-00136-of-00206.safetensors",
+ "model.layers.23.self_attn.1.q_a_proj.weight": "model-00136-of-00206.safetensors",
+ "model.layers.23.self_attn.1.q_b_proj.biases": "model-00136-of-00206.safetensors",
+ "model.layers.23.self_attn.1.q_b_proj.scales": "model-00136-of-00206.safetensors",
+ "model.layers.23.self_attn.1.q_b_proj.weight": "model-00136-of-00206.safetensors",
+ "model.layers.23.self_attn.1.unembed_out.biases": "model-00136-of-00206.safetensors",
+ "model.layers.23.self_attn.1.unembed_out.scales": "model-00136-of-00206.safetensors",
+ "model.layers.23.self_attn.1.unembed_out.weight": "model-00136-of-00206.safetensors",
+ "model.layers.24.input_layernorm.0.weight": "model-00141-of-00206.safetensors",
+ "model.layers.24.input_layernorm.1.weight": "model-00141-of-00206.safetensors",
+ "model.layers.24.mlp.router.classifier.biases": "model-00141-of-00206.safetensors",
+ "model.layers.24.mlp.router.classifier.scales": "model-00141-of-00206.safetensors",
+ "model.layers.24.mlp.router.classifier.weight": "model-00141-of-00206.safetensors",
+ "model.layers.24.mlp.router.e_score_correction_bias": "model-00141-of-00206.safetensors",
+ "model.layers.24.mlp.switch_mlp.down_proj.biases": "model-00141-of-00206.safetensors",
+ "model.layers.24.mlp.switch_mlp.down_proj.scales": "model-00140-of-00206.safetensors",
+ "model.layers.24.mlp.switch_mlp.down_proj.weight": "model-00140-of-00206.safetensors",
+ "model.layers.24.mlp.switch_mlp.gate_proj.biases": "model-00138-of-00206.safetensors",
+ "model.layers.24.mlp.switch_mlp.gate_proj.scales": "model-00137-of-00206.safetensors",
+ "model.layers.24.mlp.switch_mlp.gate_proj.weight": "model-00137-of-00206.safetensors",
+ "model.layers.24.mlp.switch_mlp.up_proj.biases": "model-00139-of-00206.safetensors",
+ "model.layers.24.mlp.switch_mlp.up_proj.scales": "model-00139-of-00206.safetensors",
+ "model.layers.24.mlp.switch_mlp.up_proj.weight": "model-00138-of-00206.safetensors",
+ "model.layers.24.mlps.0.down_proj.biases": "model-00141-of-00206.safetensors",
+ "model.layers.24.mlps.0.down_proj.scales": "model-00141-of-00206.safetensors",
+ "model.layers.24.mlps.0.down_proj.weight": "model-00141-of-00206.safetensors",
+ "model.layers.24.mlps.0.gate_proj.biases": "model-00141-of-00206.safetensors",
+ "model.layers.24.mlps.0.gate_proj.scales": "model-00141-of-00206.safetensors",
+ "model.layers.24.mlps.0.gate_proj.weight": "model-00141-of-00206.safetensors",
+ "model.layers.24.mlps.0.up_proj.biases": "model-00141-of-00206.safetensors",
+ "model.layers.24.mlps.0.up_proj.scales": "model-00141-of-00206.safetensors",
+ "model.layers.24.mlps.0.up_proj.weight": "model-00141-of-00206.safetensors",
+ "model.layers.24.mlps.1.down_proj.biases": "model-00141-of-00206.safetensors",
+ "model.layers.24.mlps.1.down_proj.scales": "model-00141-of-00206.safetensors",
+ "model.layers.24.mlps.1.down_proj.weight": "model-00141-of-00206.safetensors",
+ "model.layers.24.mlps.1.gate_proj.biases": "model-00141-of-00206.safetensors",
+ "model.layers.24.mlps.1.gate_proj.scales": "model-00141-of-00206.safetensors",
+ "model.layers.24.mlps.1.gate_proj.weight": "model-00141-of-00206.safetensors",
+ "model.layers.24.mlps.1.up_proj.biases": "model-00141-of-00206.safetensors",
+ "model.layers.24.mlps.1.up_proj.scales": "model-00141-of-00206.safetensors",
+ "model.layers.24.mlps.1.up_proj.weight": "model-00141-of-00206.safetensors",
+ "model.layers.24.post_attention_layernorm.0.weight": "model-00141-of-00206.safetensors",
+ "model.layers.24.post_attention_layernorm.1.weight": "model-00141-of-00206.safetensors",
+ "model.layers.24.self_attn.0.embed_q.biases": "model-00141-of-00206.safetensors",
+ "model.layers.24.self_attn.0.embed_q.scales": "model-00141-of-00206.safetensors",
+ "model.layers.24.self_attn.0.embed_q.weight": "model-00141-of-00206.safetensors",
+ "model.layers.24.self_attn.0.indexer.k_norm.weight": "model-00141-of-00206.safetensors",
+ "model.layers.24.self_attn.0.indexer.weights_proj.biases": "model-00141-of-00206.safetensors",
+ "model.layers.24.self_attn.0.indexer.weights_proj.scales": "model-00141-of-00206.safetensors",
+ "model.layers.24.self_attn.0.indexer.weights_proj.weight": "model-00141-of-00206.safetensors",
+ "model.layers.24.self_attn.0.indexer.wk.biases": "model-00141-of-00206.safetensors",
+ "model.layers.24.self_attn.0.indexer.wk.scales": "model-00141-of-00206.safetensors",
+ "model.layers.24.self_attn.0.indexer.wk.weight": "model-00141-of-00206.safetensors",
+ "model.layers.24.self_attn.0.indexer.wq_b.biases": "model-00141-of-00206.safetensors",
+ "model.layers.24.self_attn.0.indexer.wq_b.scales": "model-00141-of-00206.safetensors",
+ "model.layers.24.self_attn.0.indexer.wq_b.weight": "model-00141-of-00206.safetensors",
+ "model.layers.24.self_attn.0.kv_a_layernorm.weight": "model-00141-of-00206.safetensors",
+ "model.layers.24.self_attn.0.kv_a_proj_with_mqa.biases": "model-00141-of-00206.safetensors",
+ "model.layers.24.self_attn.0.kv_a_proj_with_mqa.scales": "model-00141-of-00206.safetensors",
+ "model.layers.24.self_attn.0.kv_a_proj_with_mqa.weight": "model-00141-of-00206.safetensors",
+ "model.layers.24.self_attn.0.o_proj.biases": "model-00141-of-00206.safetensors",
+ "model.layers.24.self_attn.0.o_proj.scales": "model-00141-of-00206.safetensors",
+ "model.layers.24.self_attn.0.o_proj.weight": "model-00141-of-00206.safetensors",
+ "model.layers.24.self_attn.0.q_a_layernorm.weight": "model-00141-of-00206.safetensors",
+ "model.layers.24.self_attn.0.q_a_proj.biases": "model-00141-of-00206.safetensors",
+ "model.layers.24.self_attn.0.q_a_proj.scales": "model-00141-of-00206.safetensors",
+ "model.layers.24.self_attn.0.q_a_proj.weight": "model-00141-of-00206.safetensors",
+ "model.layers.24.self_attn.0.q_b_proj.biases": "model-00141-of-00206.safetensors",
+ "model.layers.24.self_attn.0.q_b_proj.scales": "model-00141-of-00206.safetensors",
+ "model.layers.24.self_attn.0.q_b_proj.weight": "model-00141-of-00206.safetensors",
+ "model.layers.24.self_attn.0.unembed_out.biases": "model-00141-of-00206.safetensors",
+ "model.layers.24.self_attn.0.unembed_out.scales": "model-00141-of-00206.safetensors",
+ "model.layers.24.self_attn.0.unembed_out.weight": "model-00141-of-00206.safetensors",
+ "model.layers.24.self_attn.1.embed_q.biases": "model-00141-of-00206.safetensors",
+ "model.layers.24.self_attn.1.embed_q.scales": "model-00141-of-00206.safetensors",
+ "model.layers.24.self_attn.1.embed_q.weight": "model-00141-of-00206.safetensors",
+ "model.layers.24.self_attn.1.kv_a_layernorm.weight": "model-00141-of-00206.safetensors",
+ "model.layers.24.self_attn.1.kv_a_proj_with_mqa.biases": "model-00141-of-00206.safetensors",
+ "model.layers.24.self_attn.1.kv_a_proj_with_mqa.scales": "model-00141-of-00206.safetensors",
+ "model.layers.24.self_attn.1.kv_a_proj_with_mqa.weight": "model-00141-of-00206.safetensors",
+ "model.layers.24.self_attn.1.o_proj.biases": "model-00141-of-00206.safetensors",
+ "model.layers.24.self_attn.1.o_proj.scales": "model-00141-of-00206.safetensors",
+ "model.layers.24.self_attn.1.o_proj.weight": "model-00141-of-00206.safetensors",
+ "model.layers.24.self_attn.1.q_a_layernorm.weight": "model-00141-of-00206.safetensors",
+ "model.layers.24.self_attn.1.q_a_proj.biases": "model-00141-of-00206.safetensors",
+ "model.layers.24.self_attn.1.q_a_proj.scales": "model-00141-of-00206.safetensors",
+ "model.layers.24.self_attn.1.q_a_proj.weight": "model-00141-of-00206.safetensors",
+ "model.layers.24.self_attn.1.q_b_proj.biases": "model-00141-of-00206.safetensors",
+ "model.layers.24.self_attn.1.q_b_proj.scales": "model-00141-of-00206.safetensors",
+ "model.layers.24.self_attn.1.q_b_proj.weight": "model-00141-of-00206.safetensors",
+ "model.layers.24.self_attn.1.unembed_out.biases": "model-00141-of-00206.safetensors",
+ "model.layers.24.self_attn.1.unembed_out.scales": "model-00141-of-00206.safetensors",
+ "model.layers.24.self_attn.1.unembed_out.weight": "model-00141-of-00206.safetensors",
+ "model.layers.25.input_layernorm.0.weight": "model-00146-of-00206.safetensors",
+ "model.layers.25.input_layernorm.1.weight": "model-00146-of-00206.safetensors",
+ "model.layers.25.mlp.router.classifier.biases": "model-00146-of-00206.safetensors",
+ "model.layers.25.mlp.router.classifier.scales": "model-00146-of-00206.safetensors",
+ "model.layers.25.mlp.router.classifier.weight": "model-00146-of-00206.safetensors",
+ "model.layers.25.mlp.router.e_score_correction_bias": "model-00146-of-00206.safetensors",
+ "model.layers.25.mlp.switch_mlp.down_proj.biases": "model-00146-of-00206.safetensors",
+ "model.layers.25.mlp.switch_mlp.down_proj.scales": "model-00145-of-00206.safetensors",
+ "model.layers.25.mlp.switch_mlp.down_proj.weight": "model-00145-of-00206.safetensors",
+ "model.layers.25.mlp.switch_mlp.gate_proj.biases": "model-00143-of-00206.safetensors",
+ "model.layers.25.mlp.switch_mlp.gate_proj.scales": "model-00142-of-00206.safetensors",
+ "model.layers.25.mlp.switch_mlp.gate_proj.weight": "model-00142-of-00206.safetensors",
+ "model.layers.25.mlp.switch_mlp.up_proj.biases": "model-00144-of-00206.safetensors",
+ "model.layers.25.mlp.switch_mlp.up_proj.scales": "model-00144-of-00206.safetensors",
+ "model.layers.25.mlp.switch_mlp.up_proj.weight": "model-00143-of-00206.safetensors",
+ "model.layers.25.mlps.0.down_proj.biases": "model-00146-of-00206.safetensors",
+ "model.layers.25.mlps.0.down_proj.scales": "model-00146-of-00206.safetensors",
+ "model.layers.25.mlps.0.down_proj.weight": "model-00146-of-00206.safetensors",
+ "model.layers.25.mlps.0.gate_proj.biases": "model-00146-of-00206.safetensors",
+ "model.layers.25.mlps.0.gate_proj.scales": "model-00146-of-00206.safetensors",
+ "model.layers.25.mlps.0.gate_proj.weight": "model-00146-of-00206.safetensors",
+ "model.layers.25.mlps.0.up_proj.biases": "model-00146-of-00206.safetensors",
+ "model.layers.25.mlps.0.up_proj.scales": "model-00146-of-00206.safetensors",
+ "model.layers.25.mlps.0.up_proj.weight": "model-00146-of-00206.safetensors",
+ "model.layers.25.mlps.1.down_proj.biases": "model-00146-of-00206.safetensors",
+ "model.layers.25.mlps.1.down_proj.scales": "model-00146-of-00206.safetensors",
+ "model.layers.25.mlps.1.down_proj.weight": "model-00146-of-00206.safetensors",
+ "model.layers.25.mlps.1.gate_proj.biases": "model-00146-of-00206.safetensors",
+ "model.layers.25.mlps.1.gate_proj.scales": "model-00146-of-00206.safetensors",
+ "model.layers.25.mlps.1.gate_proj.weight": "model-00146-of-00206.safetensors",
+ "model.layers.25.mlps.1.up_proj.biases": "model-00146-of-00206.safetensors",
+ "model.layers.25.mlps.1.up_proj.scales": "model-00146-of-00206.safetensors",
+ "model.layers.25.mlps.1.up_proj.weight": "model-00146-of-00206.safetensors",
+ "model.layers.25.post_attention_layernorm.0.weight": "model-00146-of-00206.safetensors",
+ "model.layers.25.post_attention_layernorm.1.weight": "model-00146-of-00206.safetensors",
+ "model.layers.25.self_attn.0.embed_q.biases": "model-00146-of-00206.safetensors",
+ "model.layers.25.self_attn.0.embed_q.scales": "model-00146-of-00206.safetensors",
+ "model.layers.25.self_attn.0.embed_q.weight": "model-00146-of-00206.safetensors",
+ "model.layers.25.self_attn.0.indexer.k_norm.weight": "model-00146-of-00206.safetensors",
+ "model.layers.25.self_attn.0.indexer.weights_proj.biases": "model-00146-of-00206.safetensors",
+ "model.layers.25.self_attn.0.indexer.weights_proj.scales": "model-00146-of-00206.safetensors",
+ "model.layers.25.self_attn.0.indexer.weights_proj.weight": "model-00146-of-00206.safetensors",
+ "model.layers.25.self_attn.0.indexer.wk.biases": "model-00146-of-00206.safetensors",
+ "model.layers.25.self_attn.0.indexer.wk.scales": "model-00146-of-00206.safetensors",
+ "model.layers.25.self_attn.0.indexer.wk.weight": "model-00146-of-00206.safetensors",
+ "model.layers.25.self_attn.0.indexer.wq_b.biases": "model-00146-of-00206.safetensors",
+ "model.layers.25.self_attn.0.indexer.wq_b.scales": "model-00146-of-00206.safetensors",
+ "model.layers.25.self_attn.0.indexer.wq_b.weight": "model-00146-of-00206.safetensors",
+ "model.layers.25.self_attn.0.kv_a_layernorm.weight": "model-00146-of-00206.safetensors",
+ "model.layers.25.self_attn.0.kv_a_proj_with_mqa.biases": "model-00146-of-00206.safetensors",
+ "model.layers.25.self_attn.0.kv_a_proj_with_mqa.scales": "model-00146-of-00206.safetensors",
+ "model.layers.25.self_attn.0.kv_a_proj_with_mqa.weight": "model-00146-of-00206.safetensors",
+ "model.layers.25.self_attn.0.o_proj.biases": "model-00146-of-00206.safetensors",
+ "model.layers.25.self_attn.0.o_proj.scales": "model-00146-of-00206.safetensors",
+ "model.layers.25.self_attn.0.o_proj.weight": "model-00146-of-00206.safetensors",
+ "model.layers.25.self_attn.0.q_a_layernorm.weight": "model-00146-of-00206.safetensors",
+ "model.layers.25.self_attn.0.q_a_proj.biases": "model-00146-of-00206.safetensors",
+ "model.layers.25.self_attn.0.q_a_proj.scales": "model-00146-of-00206.safetensors",
+ "model.layers.25.self_attn.0.q_a_proj.weight": "model-00146-of-00206.safetensors",
+ "model.layers.25.self_attn.0.q_b_proj.biases": "model-00146-of-00206.safetensors",
+ "model.layers.25.self_attn.0.q_b_proj.scales": "model-00146-of-00206.safetensors",
+ "model.layers.25.self_attn.0.q_b_proj.weight": "model-00146-of-00206.safetensors",
+ "model.layers.25.self_attn.0.unembed_out.biases": "model-00146-of-00206.safetensors",
+ "model.layers.25.self_attn.0.unembed_out.scales": "model-00146-of-00206.safetensors",
+ "model.layers.25.self_attn.0.unembed_out.weight": "model-00146-of-00206.safetensors",
+ "model.layers.25.self_attn.1.embed_q.biases": "model-00146-of-00206.safetensors",
+ "model.layers.25.self_attn.1.embed_q.scales": "model-00146-of-00206.safetensors",
+ "model.layers.25.self_attn.1.embed_q.weight": "model-00146-of-00206.safetensors",
+ "model.layers.25.self_attn.1.kv_a_layernorm.weight": "model-00146-of-00206.safetensors",
+ "model.layers.25.self_attn.1.kv_a_proj_with_mqa.biases": "model-00146-of-00206.safetensors",
+ "model.layers.25.self_attn.1.kv_a_proj_with_mqa.scales": "model-00146-of-00206.safetensors",
+ "model.layers.25.self_attn.1.kv_a_proj_with_mqa.weight": "model-00146-of-00206.safetensors",
+ "model.layers.25.self_attn.1.o_proj.biases": "model-00146-of-00206.safetensors",
+ "model.layers.25.self_attn.1.o_proj.scales": "model-00146-of-00206.safetensors",
+ "model.layers.25.self_attn.1.o_proj.weight": "model-00146-of-00206.safetensors",
+ "model.layers.25.self_attn.1.q_a_layernorm.weight": "model-00146-of-00206.safetensors",
+ "model.layers.25.self_attn.1.q_a_proj.biases": "model-00146-of-00206.safetensors",
+ "model.layers.25.self_attn.1.q_a_proj.scales": "model-00146-of-00206.safetensors",
+ "model.layers.25.self_attn.1.q_a_proj.weight": "model-00146-of-00206.safetensors",
+ "model.layers.25.self_attn.1.q_b_proj.biases": "model-00146-of-00206.safetensors",
+ "model.layers.25.self_attn.1.q_b_proj.scales": "model-00146-of-00206.safetensors",
+ "model.layers.25.self_attn.1.q_b_proj.weight": "model-00146-of-00206.safetensors",
+ "model.layers.25.self_attn.1.unembed_out.biases": "model-00146-of-00206.safetensors",
+ "model.layers.25.self_attn.1.unembed_out.scales": "model-00146-of-00206.safetensors",
+ "model.layers.25.self_attn.1.unembed_out.weight": "model-00146-of-00206.safetensors",
+ "model.layers.26.input_layernorm.0.weight": "model-00151-of-00206.safetensors",
+ "model.layers.26.input_layernorm.1.weight": "model-00151-of-00206.safetensors",
+ "model.layers.26.mlp.router.classifier.biases": "model-00151-of-00206.safetensors",
+ "model.layers.26.mlp.router.classifier.scales": "model-00151-of-00206.safetensors",
+ "model.layers.26.mlp.router.classifier.weight": "model-00151-of-00206.safetensors",
+ "model.layers.26.mlp.router.e_score_correction_bias": "model-00151-of-00206.safetensors",
+ "model.layers.26.mlp.switch_mlp.down_proj.biases": "model-00151-of-00206.safetensors",
+ "model.layers.26.mlp.switch_mlp.down_proj.scales": "model-00150-of-00206.safetensors",
+ "model.layers.26.mlp.switch_mlp.down_proj.weight": "model-00150-of-00206.safetensors",
+ "model.layers.26.mlp.switch_mlp.gate_proj.biases": "model-00148-of-00206.safetensors",
+ "model.layers.26.mlp.switch_mlp.gate_proj.scales": "model-00147-of-00206.safetensors",
+ "model.layers.26.mlp.switch_mlp.gate_proj.weight": "model-00147-of-00206.safetensors",
+ "model.layers.26.mlp.switch_mlp.up_proj.biases": "model-00149-of-00206.safetensors",
+ "model.layers.26.mlp.switch_mlp.up_proj.scales": "model-00149-of-00206.safetensors",
+ "model.layers.26.mlp.switch_mlp.up_proj.weight": "model-00148-of-00206.safetensors",
+ "model.layers.26.mlps.0.down_proj.biases": "model-00151-of-00206.safetensors",
+ "model.layers.26.mlps.0.down_proj.scales": "model-00151-of-00206.safetensors",
+ "model.layers.26.mlps.0.down_proj.weight": "model-00151-of-00206.safetensors",
+ "model.layers.26.mlps.0.gate_proj.biases": "model-00151-of-00206.safetensors",
+ "model.layers.26.mlps.0.gate_proj.scales": "model-00151-of-00206.safetensors",
+ "model.layers.26.mlps.0.gate_proj.weight": "model-00151-of-00206.safetensors",
+ "model.layers.26.mlps.0.up_proj.biases": "model-00151-of-00206.safetensors",
+ "model.layers.26.mlps.0.up_proj.scales": "model-00151-of-00206.safetensors",
+ "model.layers.26.mlps.0.up_proj.weight": "model-00151-of-00206.safetensors",
+ "model.layers.26.mlps.1.down_proj.biases": "model-00151-of-00206.safetensors",
+ "model.layers.26.mlps.1.down_proj.scales": "model-00151-of-00206.safetensors",
+ "model.layers.26.mlps.1.down_proj.weight": "model-00151-of-00206.safetensors",
+ "model.layers.26.mlps.1.gate_proj.biases": "model-00151-of-00206.safetensors",
+ "model.layers.26.mlps.1.gate_proj.scales": "model-00151-of-00206.safetensors",
+ "model.layers.26.mlps.1.gate_proj.weight": "model-00151-of-00206.safetensors",
+ "model.layers.26.mlps.1.up_proj.biases": "model-00151-of-00206.safetensors",
+ "model.layers.26.mlps.1.up_proj.scales": "model-00151-of-00206.safetensors",
+ "model.layers.26.mlps.1.up_proj.weight": "model-00151-of-00206.safetensors",
+ "model.layers.26.post_attention_layernorm.0.weight": "model-00151-of-00206.safetensors",
+ "model.layers.26.post_attention_layernorm.1.weight": "model-00151-of-00206.safetensors",
+ "model.layers.26.self_attn.0.embed_q.biases": "model-00151-of-00206.safetensors",
+ "model.layers.26.self_attn.0.embed_q.scales": "model-00151-of-00206.safetensors",
+ "model.layers.26.self_attn.0.embed_q.weight": "model-00151-of-00206.safetensors",
+ "model.layers.26.self_attn.0.indexer.k_norm.weight": "model-00151-of-00206.safetensors",
+ "model.layers.26.self_attn.0.indexer.weights_proj.biases": "model-00151-of-00206.safetensors",
+ "model.layers.26.self_attn.0.indexer.weights_proj.scales": "model-00151-of-00206.safetensors",
+ "model.layers.26.self_attn.0.indexer.weights_proj.weight": "model-00151-of-00206.safetensors",
+ "model.layers.26.self_attn.0.indexer.wk.biases": "model-00151-of-00206.safetensors",
+ "model.layers.26.self_attn.0.indexer.wk.scales": "model-00151-of-00206.safetensors",
+ "model.layers.26.self_attn.0.indexer.wk.weight": "model-00151-of-00206.safetensors",
+ "model.layers.26.self_attn.0.indexer.wq_b.biases": "model-00151-of-00206.safetensors",
+ "model.layers.26.self_attn.0.indexer.wq_b.scales": "model-00151-of-00206.safetensors",
+ "model.layers.26.self_attn.0.indexer.wq_b.weight": "model-00151-of-00206.safetensors",
+ "model.layers.26.self_attn.0.kv_a_layernorm.weight": "model-00151-of-00206.safetensors",
+ "model.layers.26.self_attn.0.kv_a_proj_with_mqa.biases": "model-00151-of-00206.safetensors",
+ "model.layers.26.self_attn.0.kv_a_proj_with_mqa.scales": "model-00151-of-00206.safetensors",
+ "model.layers.26.self_attn.0.kv_a_proj_with_mqa.weight": "model-00151-of-00206.safetensors",
+ "model.layers.26.self_attn.0.o_proj.biases": "model-00151-of-00206.safetensors",
+ "model.layers.26.self_attn.0.o_proj.scales": "model-00151-of-00206.safetensors",
+ "model.layers.26.self_attn.0.o_proj.weight": "model-00151-of-00206.safetensors",
+ "model.layers.26.self_attn.0.q_a_layernorm.weight": "model-00151-of-00206.safetensors",
+ "model.layers.26.self_attn.0.q_a_proj.biases": "model-00151-of-00206.safetensors",
+ "model.layers.26.self_attn.0.q_a_proj.scales": "model-00151-of-00206.safetensors",
+ "model.layers.26.self_attn.0.q_a_proj.weight": "model-00151-of-00206.safetensors",
+ "model.layers.26.self_attn.0.q_b_proj.biases": "model-00151-of-00206.safetensors",
+ "model.layers.26.self_attn.0.q_b_proj.scales": "model-00151-of-00206.safetensors",
+ "model.layers.26.self_attn.0.q_b_proj.weight": "model-00151-of-00206.safetensors",
+ "model.layers.26.self_attn.0.unembed_out.biases": "model-00151-of-00206.safetensors",
+ "model.layers.26.self_attn.0.unembed_out.scales": "model-00151-of-00206.safetensors",
+ "model.layers.26.self_attn.0.unembed_out.weight": "model-00151-of-00206.safetensors",
+ "model.layers.26.self_attn.1.embed_q.biases": "model-00151-of-00206.safetensors",
+ "model.layers.26.self_attn.1.embed_q.scales": "model-00151-of-00206.safetensors",
+ "model.layers.26.self_attn.1.embed_q.weight": "model-00151-of-00206.safetensors",
+ "model.layers.26.self_attn.1.kv_a_layernorm.weight": "model-00151-of-00206.safetensors",
+ "model.layers.26.self_attn.1.kv_a_proj_with_mqa.biases": "model-00151-of-00206.safetensors",
+ "model.layers.26.self_attn.1.kv_a_proj_with_mqa.scales": "model-00151-of-00206.safetensors",
+ "model.layers.26.self_attn.1.kv_a_proj_with_mqa.weight": "model-00151-of-00206.safetensors",
+ "model.layers.26.self_attn.1.o_proj.biases": "model-00151-of-00206.safetensors",
+ "model.layers.26.self_attn.1.o_proj.scales": "model-00151-of-00206.safetensors",
+ "model.layers.26.self_attn.1.o_proj.weight": "model-00151-of-00206.safetensors",
+ "model.layers.26.self_attn.1.q_a_layernorm.weight": "model-00151-of-00206.safetensors",
+ "model.layers.26.self_attn.1.q_a_proj.biases": "model-00151-of-00206.safetensors",
+ "model.layers.26.self_attn.1.q_a_proj.scales": "model-00151-of-00206.safetensors",
+ "model.layers.26.self_attn.1.q_a_proj.weight": "model-00151-of-00206.safetensors",
+ "model.layers.26.self_attn.1.q_b_proj.biases": "model-00151-of-00206.safetensors",
+ "model.layers.26.self_attn.1.q_b_proj.scales": "model-00151-of-00206.safetensors",
+ "model.layers.26.self_attn.1.q_b_proj.weight": "model-00151-of-00206.safetensors",
+ "model.layers.26.self_attn.1.unembed_out.biases": "model-00151-of-00206.safetensors",
+ "model.layers.26.self_attn.1.unembed_out.scales": "model-00151-of-00206.safetensors",
+ "model.layers.26.self_attn.1.unembed_out.weight": "model-00151-of-00206.safetensors",
+ "model.layers.27.input_layernorm.0.weight": "model-00156-of-00206.safetensors",
+ "model.layers.27.input_layernorm.1.weight": "model-00156-of-00206.safetensors",
+ "model.layers.27.mlp.router.classifier.biases": "model-00156-of-00206.safetensors",
+ "model.layers.27.mlp.router.classifier.scales": "model-00156-of-00206.safetensors",
+ "model.layers.27.mlp.router.classifier.weight": "model-00156-of-00206.safetensors",
+ "model.layers.27.mlp.router.e_score_correction_bias": "model-00156-of-00206.safetensors",
+ "model.layers.27.mlp.switch_mlp.down_proj.biases": "model-00156-of-00206.safetensors",
+ "model.layers.27.mlp.switch_mlp.down_proj.scales": "model-00155-of-00206.safetensors",
+ "model.layers.27.mlp.switch_mlp.down_proj.weight": "model-00155-of-00206.safetensors",
+ "model.layers.27.mlp.switch_mlp.gate_proj.biases": "model-00153-of-00206.safetensors",
+ "model.layers.27.mlp.switch_mlp.gate_proj.scales": "model-00152-of-00206.safetensors",
+ "model.layers.27.mlp.switch_mlp.gate_proj.weight": "model-00152-of-00206.safetensors",
+ "model.layers.27.mlp.switch_mlp.up_proj.biases": "model-00154-of-00206.safetensors",
+ "model.layers.27.mlp.switch_mlp.up_proj.scales": "model-00154-of-00206.safetensors",
+ "model.layers.27.mlp.switch_mlp.up_proj.weight": "model-00153-of-00206.safetensors",
+ "model.layers.27.mlps.0.down_proj.biases": "model-00156-of-00206.safetensors",
+ "model.layers.27.mlps.0.down_proj.scales": "model-00156-of-00206.safetensors",
+ "model.layers.27.mlps.0.down_proj.weight": "model-00156-of-00206.safetensors",
+ "model.layers.27.mlps.0.gate_proj.biases": "model-00156-of-00206.safetensors",
+ "model.layers.27.mlps.0.gate_proj.scales": "model-00156-of-00206.safetensors",
+ "model.layers.27.mlps.0.gate_proj.weight": "model-00156-of-00206.safetensors",
+ "model.layers.27.mlps.0.up_proj.biases": "model-00156-of-00206.safetensors",
+ "model.layers.27.mlps.0.up_proj.scales": "model-00156-of-00206.safetensors",
+ "model.layers.27.mlps.0.up_proj.weight": "model-00156-of-00206.safetensors",
+ "model.layers.27.mlps.1.down_proj.biases": "model-00156-of-00206.safetensors",
+ "model.layers.27.mlps.1.down_proj.scales": "model-00156-of-00206.safetensors",
+ "model.layers.27.mlps.1.down_proj.weight": "model-00156-of-00206.safetensors",
+ "model.layers.27.mlps.1.gate_proj.biases": "model-00156-of-00206.safetensors",
+ "model.layers.27.mlps.1.gate_proj.scales": "model-00156-of-00206.safetensors",
+ "model.layers.27.mlps.1.gate_proj.weight": "model-00156-of-00206.safetensors",
+ "model.layers.27.mlps.1.up_proj.biases": "model-00156-of-00206.safetensors",
+ "model.layers.27.mlps.1.up_proj.scales": "model-00156-of-00206.safetensors",
+ "model.layers.27.mlps.1.up_proj.weight": "model-00156-of-00206.safetensors",
+ "model.layers.27.post_attention_layernorm.0.weight": "model-00156-of-00206.safetensors",
+ "model.layers.27.post_attention_layernorm.1.weight": "model-00156-of-00206.safetensors",
+ "model.layers.27.self_attn.0.embed_q.biases": "model-00156-of-00206.safetensors",
+ "model.layers.27.self_attn.0.embed_q.scales": "model-00156-of-00206.safetensors",
+ "model.layers.27.self_attn.0.embed_q.weight": "model-00156-of-00206.safetensors",
+ "model.layers.27.self_attn.0.indexer.k_norm.weight": "model-00156-of-00206.safetensors",
+ "model.layers.27.self_attn.0.indexer.weights_proj.biases": "model-00156-of-00206.safetensors",
+ "model.layers.27.self_attn.0.indexer.weights_proj.scales": "model-00156-of-00206.safetensors",
+ "model.layers.27.self_attn.0.indexer.weights_proj.weight": "model-00156-of-00206.safetensors",
+ "model.layers.27.self_attn.0.indexer.wk.biases": "model-00156-of-00206.safetensors",
+ "model.layers.27.self_attn.0.indexer.wk.scales": "model-00156-of-00206.safetensors",
+ "model.layers.27.self_attn.0.indexer.wk.weight": "model-00156-of-00206.safetensors",
+ "model.layers.27.self_attn.0.indexer.wq_b.biases": "model-00156-of-00206.safetensors",
+ "model.layers.27.self_attn.0.indexer.wq_b.scales": "model-00156-of-00206.safetensors",
+ "model.layers.27.self_attn.0.indexer.wq_b.weight": "model-00156-of-00206.safetensors",
+ "model.layers.27.self_attn.0.kv_a_layernorm.weight": "model-00156-of-00206.safetensors",
+ "model.layers.27.self_attn.0.kv_a_proj_with_mqa.biases": "model-00156-of-00206.safetensors",
+ "model.layers.27.self_attn.0.kv_a_proj_with_mqa.scales": "model-00156-of-00206.safetensors",
+ "model.layers.27.self_attn.0.kv_a_proj_with_mqa.weight": "model-00156-of-00206.safetensors",
+ "model.layers.27.self_attn.0.o_proj.biases": "model-00156-of-00206.safetensors",
+ "model.layers.27.self_attn.0.o_proj.scales": "model-00156-of-00206.safetensors",
+ "model.layers.27.self_attn.0.o_proj.weight": "model-00156-of-00206.safetensors",
+ "model.layers.27.self_attn.0.q_a_layernorm.weight": "model-00156-of-00206.safetensors",
+ "model.layers.27.self_attn.0.q_a_proj.biases": "model-00156-of-00206.safetensors",
+ "model.layers.27.self_attn.0.q_a_proj.scales": "model-00156-of-00206.safetensors",
+ "model.layers.27.self_attn.0.q_a_proj.weight": "model-00156-of-00206.safetensors",
+ "model.layers.27.self_attn.0.q_b_proj.biases": "model-00156-of-00206.safetensors",
+ "model.layers.27.self_attn.0.q_b_proj.scales": "model-00156-of-00206.safetensors",
+ "model.layers.27.self_attn.0.q_b_proj.weight": "model-00156-of-00206.safetensors",
+ "model.layers.27.self_attn.0.unembed_out.biases": "model-00156-of-00206.safetensors",
+ "model.layers.27.self_attn.0.unembed_out.scales": "model-00156-of-00206.safetensors",
+ "model.layers.27.self_attn.0.unembed_out.weight": "model-00156-of-00206.safetensors",
+ "model.layers.27.self_attn.1.embed_q.biases": "model-00156-of-00206.safetensors",
+ "model.layers.27.self_attn.1.embed_q.scales": "model-00156-of-00206.safetensors",
+ "model.layers.27.self_attn.1.embed_q.weight": "model-00156-of-00206.safetensors",
+ "model.layers.27.self_attn.1.kv_a_layernorm.weight": "model-00156-of-00206.safetensors",
+ "model.layers.27.self_attn.1.kv_a_proj_with_mqa.biases": "model-00156-of-00206.safetensors",
+ "model.layers.27.self_attn.1.kv_a_proj_with_mqa.scales": "model-00156-of-00206.safetensors",
+ "model.layers.27.self_attn.1.kv_a_proj_with_mqa.weight": "model-00156-of-00206.safetensors",
+ "model.layers.27.self_attn.1.o_proj.biases": "model-00156-of-00206.safetensors",
+ "model.layers.27.self_attn.1.o_proj.scales": "model-00156-of-00206.safetensors",
+ "model.layers.27.self_attn.1.o_proj.weight": "model-00156-of-00206.safetensors",
+ "model.layers.27.self_attn.1.q_a_layernorm.weight": "model-00156-of-00206.safetensors",
+ "model.layers.27.self_attn.1.q_a_proj.biases": "model-00156-of-00206.safetensors",
+ "model.layers.27.self_attn.1.q_a_proj.scales": "model-00156-of-00206.safetensors",
+ "model.layers.27.self_attn.1.q_a_proj.weight": "model-00156-of-00206.safetensors",
+ "model.layers.27.self_attn.1.q_b_proj.biases": "model-00156-of-00206.safetensors",
+ "model.layers.27.self_attn.1.q_b_proj.scales": "model-00156-of-00206.safetensors",
+ "model.layers.27.self_attn.1.q_b_proj.weight": "model-00156-of-00206.safetensors",
+ "model.layers.27.self_attn.1.unembed_out.biases": "model-00156-of-00206.safetensors",
+ "model.layers.27.self_attn.1.unembed_out.scales": "model-00156-of-00206.safetensors",
+ "model.layers.27.self_attn.1.unembed_out.weight": "model-00156-of-00206.safetensors",
+ "model.layers.28.input_layernorm.0.weight": "model-00161-of-00206.safetensors",
+ "model.layers.28.input_layernorm.1.weight": "model-00161-of-00206.safetensors",
+ "model.layers.28.mlp.router.classifier.biases": "model-00161-of-00206.safetensors",
+ "model.layers.28.mlp.router.classifier.scales": "model-00161-of-00206.safetensors",
+ "model.layers.28.mlp.router.classifier.weight": "model-00161-of-00206.safetensors",
+ "model.layers.28.mlp.router.e_score_correction_bias": "model-00161-of-00206.safetensors",
+ "model.layers.28.mlp.switch_mlp.down_proj.biases": "model-00161-of-00206.safetensors",
+ "model.layers.28.mlp.switch_mlp.down_proj.scales": "model-00160-of-00206.safetensors",
+ "model.layers.28.mlp.switch_mlp.down_proj.weight": "model-00160-of-00206.safetensors",
+ "model.layers.28.mlp.switch_mlp.gate_proj.biases": "model-00158-of-00206.safetensors",
+ "model.layers.28.mlp.switch_mlp.gate_proj.scales": "model-00157-of-00206.safetensors",
+ "model.layers.28.mlp.switch_mlp.gate_proj.weight": "model-00157-of-00206.safetensors",
+ "model.layers.28.mlp.switch_mlp.up_proj.biases": "model-00159-of-00206.safetensors",
+ "model.layers.28.mlp.switch_mlp.up_proj.scales": "model-00159-of-00206.safetensors",
+ "model.layers.28.mlp.switch_mlp.up_proj.weight": "model-00158-of-00206.safetensors",
+ "model.layers.28.mlps.0.down_proj.biases": "model-00161-of-00206.safetensors",
+ "model.layers.28.mlps.0.down_proj.scales": "model-00161-of-00206.safetensors",
+ "model.layers.28.mlps.0.down_proj.weight": "model-00161-of-00206.safetensors",
+ "model.layers.28.mlps.0.gate_proj.biases": "model-00161-of-00206.safetensors",
+ "model.layers.28.mlps.0.gate_proj.scales": "model-00161-of-00206.safetensors",
+ "model.layers.28.mlps.0.gate_proj.weight": "model-00161-of-00206.safetensors",
+ "model.layers.28.mlps.0.up_proj.biases": "model-00161-of-00206.safetensors",
+ "model.layers.28.mlps.0.up_proj.scales": "model-00161-of-00206.safetensors",
+ "model.layers.28.mlps.0.up_proj.weight": "model-00161-of-00206.safetensors",
+ "model.layers.28.mlps.1.down_proj.biases": "model-00161-of-00206.safetensors",
+ "model.layers.28.mlps.1.down_proj.scales": "model-00161-of-00206.safetensors",
+ "model.layers.28.mlps.1.down_proj.weight": "model-00161-of-00206.safetensors",
+ "model.layers.28.mlps.1.gate_proj.biases": "model-00161-of-00206.safetensors",
+ "model.layers.28.mlps.1.gate_proj.scales": "model-00161-of-00206.safetensors",
+ "model.layers.28.mlps.1.gate_proj.weight": "model-00161-of-00206.safetensors",
+ "model.layers.28.mlps.1.up_proj.biases": "model-00161-of-00206.safetensors",
+ "model.layers.28.mlps.1.up_proj.scales": "model-00161-of-00206.safetensors",
+ "model.layers.28.mlps.1.up_proj.weight": "model-00161-of-00206.safetensors",
+ "model.layers.28.post_attention_layernorm.0.weight": "model-00161-of-00206.safetensors",
+ "model.layers.28.post_attention_layernorm.1.weight": "model-00161-of-00206.safetensors",
+ "model.layers.28.self_attn.0.embed_q.biases": "model-00161-of-00206.safetensors",
+ "model.layers.28.self_attn.0.embed_q.scales": "model-00161-of-00206.safetensors",
+ "model.layers.28.self_attn.0.embed_q.weight": "model-00161-of-00206.safetensors",
+ "model.layers.28.self_attn.0.indexer.k_norm.weight": "model-00161-of-00206.safetensors",
+ "model.layers.28.self_attn.0.indexer.weights_proj.biases": "model-00161-of-00206.safetensors",
+ "model.layers.28.self_attn.0.indexer.weights_proj.scales": "model-00161-of-00206.safetensors",
+ "model.layers.28.self_attn.0.indexer.weights_proj.weight": "model-00161-of-00206.safetensors",
+ "model.layers.28.self_attn.0.indexer.wk.biases": "model-00161-of-00206.safetensors",
+ "model.layers.28.self_attn.0.indexer.wk.scales": "model-00161-of-00206.safetensors",
+ "model.layers.28.self_attn.0.indexer.wk.weight": "model-00161-of-00206.safetensors",
+ "model.layers.28.self_attn.0.indexer.wq_b.biases": "model-00161-of-00206.safetensors",
+ "model.layers.28.self_attn.0.indexer.wq_b.scales": "model-00161-of-00206.safetensors",
+ "model.layers.28.self_attn.0.indexer.wq_b.weight": "model-00161-of-00206.safetensors",
+ "model.layers.28.self_attn.0.kv_a_layernorm.weight": "model-00161-of-00206.safetensors",
+ "model.layers.28.self_attn.0.kv_a_proj_with_mqa.biases": "model-00161-of-00206.safetensors",
+ "model.layers.28.self_attn.0.kv_a_proj_with_mqa.scales": "model-00161-of-00206.safetensors",
+ "model.layers.28.self_attn.0.kv_a_proj_with_mqa.weight": "model-00161-of-00206.safetensors",
+ "model.layers.28.self_attn.0.o_proj.biases": "model-00161-of-00206.safetensors",
+ "model.layers.28.self_attn.0.o_proj.scales": "model-00161-of-00206.safetensors",
+ "model.layers.28.self_attn.0.o_proj.weight": "model-00161-of-00206.safetensors",
+ "model.layers.28.self_attn.0.q_a_layernorm.weight": "model-00161-of-00206.safetensors",
+ "model.layers.28.self_attn.0.q_a_proj.biases": "model-00161-of-00206.safetensors",
+ "model.layers.28.self_attn.0.q_a_proj.scales": "model-00161-of-00206.safetensors",
+ "model.layers.28.self_attn.0.q_a_proj.weight": "model-00161-of-00206.safetensors",
+ "model.layers.28.self_attn.0.q_b_proj.biases": "model-00161-of-00206.safetensors",
+ "model.layers.28.self_attn.0.q_b_proj.scales": "model-00161-of-00206.safetensors",
+ "model.layers.28.self_attn.0.q_b_proj.weight": "model-00161-of-00206.safetensors",
+ "model.layers.28.self_attn.0.unembed_out.biases": "model-00161-of-00206.safetensors",
+ "model.layers.28.self_attn.0.unembed_out.scales": "model-00161-of-00206.safetensors",
+ "model.layers.28.self_attn.0.unembed_out.weight": "model-00161-of-00206.safetensors",
+ "model.layers.28.self_attn.1.embed_q.biases": "model-00161-of-00206.safetensors",
+ "model.layers.28.self_attn.1.embed_q.scales": "model-00161-of-00206.safetensors",
+ "model.layers.28.self_attn.1.embed_q.weight": "model-00161-of-00206.safetensors",
+ "model.layers.28.self_attn.1.kv_a_layernorm.weight": "model-00161-of-00206.safetensors",
+ "model.layers.28.self_attn.1.kv_a_proj_with_mqa.biases": "model-00161-of-00206.safetensors",
+ "model.layers.28.self_attn.1.kv_a_proj_with_mqa.scales": "model-00161-of-00206.safetensors",
+ "model.layers.28.self_attn.1.kv_a_proj_with_mqa.weight": "model-00161-of-00206.safetensors",
+ "model.layers.28.self_attn.1.o_proj.biases": "model-00161-of-00206.safetensors",
+ "model.layers.28.self_attn.1.o_proj.scales": "model-00161-of-00206.safetensors",
+ "model.layers.28.self_attn.1.o_proj.weight": "model-00161-of-00206.safetensors",
+ "model.layers.28.self_attn.1.q_a_layernorm.weight": "model-00161-of-00206.safetensors",
+ "model.layers.28.self_attn.1.q_a_proj.biases": "model-00161-of-00206.safetensors",
+ "model.layers.28.self_attn.1.q_a_proj.scales": "model-00161-of-00206.safetensors",
+ "model.layers.28.self_attn.1.q_a_proj.weight": "model-00161-of-00206.safetensors",
+ "model.layers.28.self_attn.1.q_b_proj.biases": "model-00161-of-00206.safetensors",
+ "model.layers.28.self_attn.1.q_b_proj.scales": "model-00161-of-00206.safetensors",
+ "model.layers.28.self_attn.1.q_b_proj.weight": "model-00161-of-00206.safetensors",
+ "model.layers.28.self_attn.1.unembed_out.biases": "model-00161-of-00206.safetensors",
+ "model.layers.28.self_attn.1.unembed_out.scales": "model-00161-of-00206.safetensors",
+ "model.layers.28.self_attn.1.unembed_out.weight": "model-00161-of-00206.safetensors",
+ "model.layers.29.input_layernorm.0.weight": "model-00166-of-00206.safetensors",
+ "model.layers.29.input_layernorm.1.weight": "model-00166-of-00206.safetensors",
+ "model.layers.29.mlp.router.classifier.biases": "model-00166-of-00206.safetensors",
+ "model.layers.29.mlp.router.classifier.scales": "model-00166-of-00206.safetensors",
+ "model.layers.29.mlp.router.classifier.weight": "model-00166-of-00206.safetensors",
+ "model.layers.29.mlp.router.e_score_correction_bias": "model-00166-of-00206.safetensors",
+ "model.layers.29.mlp.switch_mlp.down_proj.biases": "model-00166-of-00206.safetensors",
+ "model.layers.29.mlp.switch_mlp.down_proj.scales": "model-00165-of-00206.safetensors",
+ "model.layers.29.mlp.switch_mlp.down_proj.weight": "model-00165-of-00206.safetensors",
+ "model.layers.29.mlp.switch_mlp.gate_proj.biases": "model-00163-of-00206.safetensors",
+ "model.layers.29.mlp.switch_mlp.gate_proj.scales": "model-00162-of-00206.safetensors",
+ "model.layers.29.mlp.switch_mlp.gate_proj.weight": "model-00162-of-00206.safetensors",
+ "model.layers.29.mlp.switch_mlp.up_proj.biases": "model-00164-of-00206.safetensors",
+ "model.layers.29.mlp.switch_mlp.up_proj.scales": "model-00164-of-00206.safetensors",
+ "model.layers.29.mlp.switch_mlp.up_proj.weight": "model-00163-of-00206.safetensors",
+ "model.layers.29.mlps.0.down_proj.biases": "model-00166-of-00206.safetensors",
+ "model.layers.29.mlps.0.down_proj.scales": "model-00166-of-00206.safetensors",
+ "model.layers.29.mlps.0.down_proj.weight": "model-00166-of-00206.safetensors",
+ "model.layers.29.mlps.0.gate_proj.biases": "model-00166-of-00206.safetensors",
+ "model.layers.29.mlps.0.gate_proj.scales": "model-00166-of-00206.safetensors",
+ "model.layers.29.mlps.0.gate_proj.weight": "model-00166-of-00206.safetensors",
+ "model.layers.29.mlps.0.up_proj.biases": "model-00166-of-00206.safetensors",
+ "model.layers.29.mlps.0.up_proj.scales": "model-00166-of-00206.safetensors",
+ "model.layers.29.mlps.0.up_proj.weight": "model-00166-of-00206.safetensors",
+ "model.layers.29.mlps.1.down_proj.biases": "model-00166-of-00206.safetensors",
+ "model.layers.29.mlps.1.down_proj.scales": "model-00166-of-00206.safetensors",
+ "model.layers.29.mlps.1.down_proj.weight": "model-00166-of-00206.safetensors",
+ "model.layers.29.mlps.1.gate_proj.biases": "model-00166-of-00206.safetensors",
+ "model.layers.29.mlps.1.gate_proj.scales": "model-00166-of-00206.safetensors",
+ "model.layers.29.mlps.1.gate_proj.weight": "model-00166-of-00206.safetensors",
+ "model.layers.29.mlps.1.up_proj.biases": "model-00166-of-00206.safetensors",
+ "model.layers.29.mlps.1.up_proj.scales": "model-00166-of-00206.safetensors",
+ "model.layers.29.mlps.1.up_proj.weight": "model-00166-of-00206.safetensors",
+ "model.layers.29.post_attention_layernorm.0.weight": "model-00166-of-00206.safetensors",
+ "model.layers.29.post_attention_layernorm.1.weight": "model-00166-of-00206.safetensors",
+ "model.layers.29.self_attn.0.embed_q.biases": "model-00166-of-00206.safetensors",
+ "model.layers.29.self_attn.0.embed_q.scales": "model-00166-of-00206.safetensors",
+ "model.layers.29.self_attn.0.embed_q.weight": "model-00166-of-00206.safetensors",
+ "model.layers.29.self_attn.0.indexer.k_norm.weight": "model-00166-of-00206.safetensors",
+ "model.layers.29.self_attn.0.indexer.weights_proj.biases": "model-00166-of-00206.safetensors",
+ "model.layers.29.self_attn.0.indexer.weights_proj.scales": "model-00166-of-00206.safetensors",
+ "model.layers.29.self_attn.0.indexer.weights_proj.weight": "model-00166-of-00206.safetensors",
+ "model.layers.29.self_attn.0.indexer.wk.biases": "model-00166-of-00206.safetensors",
+ "model.layers.29.self_attn.0.indexer.wk.scales": "model-00166-of-00206.safetensors",
+ "model.layers.29.self_attn.0.indexer.wk.weight": "model-00166-of-00206.safetensors",
+ "model.layers.29.self_attn.0.indexer.wq_b.biases": "model-00166-of-00206.safetensors",
+ "model.layers.29.self_attn.0.indexer.wq_b.scales": "model-00166-of-00206.safetensors",
+ "model.layers.29.self_attn.0.indexer.wq_b.weight": "model-00166-of-00206.safetensors",
+ "model.layers.29.self_attn.0.kv_a_layernorm.weight": "model-00166-of-00206.safetensors",
+ "model.layers.29.self_attn.0.kv_a_proj_with_mqa.biases": "model-00166-of-00206.safetensors",
+ "model.layers.29.self_attn.0.kv_a_proj_with_mqa.scales": "model-00166-of-00206.safetensors",
+ "model.layers.29.self_attn.0.kv_a_proj_with_mqa.weight": "model-00166-of-00206.safetensors",
+ "model.layers.29.self_attn.0.o_proj.biases": "model-00166-of-00206.safetensors",
+ "model.layers.29.self_attn.0.o_proj.scales": "model-00166-of-00206.safetensors",
+ "model.layers.29.self_attn.0.o_proj.weight": "model-00166-of-00206.safetensors",
+ "model.layers.29.self_attn.0.q_a_layernorm.weight": "model-00166-of-00206.safetensors",
+ "model.layers.29.self_attn.0.q_a_proj.biases": "model-00166-of-00206.safetensors",
+ "model.layers.29.self_attn.0.q_a_proj.scales": "model-00166-of-00206.safetensors",
+ "model.layers.29.self_attn.0.q_a_proj.weight": "model-00166-of-00206.safetensors",
+ "model.layers.29.self_attn.0.q_b_proj.biases": "model-00166-of-00206.safetensors",
+ "model.layers.29.self_attn.0.q_b_proj.scales": "model-00166-of-00206.safetensors",
+ "model.layers.29.self_attn.0.q_b_proj.weight": "model-00166-of-00206.safetensors",
+ "model.layers.29.self_attn.0.unembed_out.biases": "model-00166-of-00206.safetensors",
+ "model.layers.29.self_attn.0.unembed_out.scales": "model-00166-of-00206.safetensors",
+ "model.layers.29.self_attn.0.unembed_out.weight": "model-00166-of-00206.safetensors",
+ "model.layers.29.self_attn.1.embed_q.biases": "model-00166-of-00206.safetensors",
+ "model.layers.29.self_attn.1.embed_q.scales": "model-00166-of-00206.safetensors",
+ "model.layers.29.self_attn.1.embed_q.weight": "model-00166-of-00206.safetensors",
+ "model.layers.29.self_attn.1.kv_a_layernorm.weight": "model-00166-of-00206.safetensors",
+ "model.layers.29.self_attn.1.kv_a_proj_with_mqa.biases": "model-00166-of-00206.safetensors",
+ "model.layers.29.self_attn.1.kv_a_proj_with_mqa.scales": "model-00166-of-00206.safetensors",
+ "model.layers.29.self_attn.1.kv_a_proj_with_mqa.weight": "model-00166-of-00206.safetensors",
+ "model.layers.29.self_attn.1.o_proj.biases": "model-00166-of-00206.safetensors",
+ "model.layers.29.self_attn.1.o_proj.scales": "model-00166-of-00206.safetensors",
+ "model.layers.29.self_attn.1.o_proj.weight": "model-00166-of-00206.safetensors",
+ "model.layers.29.self_attn.1.q_a_layernorm.weight": "model-00166-of-00206.safetensors",
+ "model.layers.29.self_attn.1.q_a_proj.biases": "model-00166-of-00206.safetensors",
+ "model.layers.29.self_attn.1.q_a_proj.scales": "model-00166-of-00206.safetensors",
+ "model.layers.29.self_attn.1.q_a_proj.weight": "model-00166-of-00206.safetensors",
+ "model.layers.29.self_attn.1.q_b_proj.biases": "model-00166-of-00206.safetensors",
+ "model.layers.29.self_attn.1.q_b_proj.scales": "model-00166-of-00206.safetensors",
+ "model.layers.29.self_attn.1.q_b_proj.weight": "model-00166-of-00206.safetensors",
+ "model.layers.29.self_attn.1.unembed_out.biases": "model-00166-of-00206.safetensors",
+ "model.layers.29.self_attn.1.unembed_out.scales": "model-00166-of-00206.safetensors",
+ "model.layers.29.self_attn.1.unembed_out.weight": "model-00166-of-00206.safetensors",
+ "model.layers.3.input_layernorm.0.weight": "model-00036-of-00206.safetensors",
+ "model.layers.3.input_layernorm.1.weight": "model-00036-of-00206.safetensors",
+ "model.layers.3.mlp.router.classifier.biases": "model-00036-of-00206.safetensors",
+ "model.layers.3.mlp.router.classifier.scales": "model-00036-of-00206.safetensors",
+ "model.layers.3.mlp.router.classifier.weight": "model-00036-of-00206.safetensors",
+ "model.layers.3.mlp.router.e_score_correction_bias": "model-00036-of-00206.safetensors",
+ "model.layers.3.mlp.switch_mlp.down_proj.biases": "model-00036-of-00206.safetensors",
+ "model.layers.3.mlp.switch_mlp.down_proj.scales": "model-00035-of-00206.safetensors",
+ "model.layers.3.mlp.switch_mlp.down_proj.weight": "model-00035-of-00206.safetensors",
+ "model.layers.3.mlp.switch_mlp.gate_proj.biases": "model-00033-of-00206.safetensors",
+ "model.layers.3.mlp.switch_mlp.gate_proj.scales": "model-00032-of-00206.safetensors",
+ "model.layers.3.mlp.switch_mlp.gate_proj.weight": "model-00032-of-00206.safetensors",
+ "model.layers.3.mlp.switch_mlp.up_proj.biases": "model-00034-of-00206.safetensors",
+ "model.layers.3.mlp.switch_mlp.up_proj.scales": "model-00034-of-00206.safetensors",
+ "model.layers.3.mlp.switch_mlp.up_proj.weight": "model-00033-of-00206.safetensors",
+ "model.layers.3.mlps.0.down_proj.biases": "model-00036-of-00206.safetensors",
+ "model.layers.3.mlps.0.down_proj.scales": "model-00036-of-00206.safetensors",
+ "model.layers.3.mlps.0.down_proj.weight": "model-00036-of-00206.safetensors",
+ "model.layers.3.mlps.0.gate_proj.biases": "model-00036-of-00206.safetensors",
+ "model.layers.3.mlps.0.gate_proj.scales": "model-00036-of-00206.safetensors",
+ "model.layers.3.mlps.0.gate_proj.weight": "model-00036-of-00206.safetensors",
+ "model.layers.3.mlps.0.up_proj.biases": "model-00036-of-00206.safetensors",
+ "model.layers.3.mlps.0.up_proj.scales": "model-00036-of-00206.safetensors",
+ "model.layers.3.mlps.0.up_proj.weight": "model-00036-of-00206.safetensors",
+ "model.layers.3.mlps.1.down_proj.biases": "model-00036-of-00206.safetensors",
+ "model.layers.3.mlps.1.down_proj.scales": "model-00036-of-00206.safetensors",
+ "model.layers.3.mlps.1.down_proj.weight": "model-00036-of-00206.safetensors",
+ "model.layers.3.mlps.1.gate_proj.biases": "model-00036-of-00206.safetensors",
+ "model.layers.3.mlps.1.gate_proj.scales": "model-00036-of-00206.safetensors",
+ "model.layers.3.mlps.1.gate_proj.weight": "model-00036-of-00206.safetensors",
+ "model.layers.3.mlps.1.up_proj.biases": "model-00036-of-00206.safetensors",
+ "model.layers.3.mlps.1.up_proj.scales": "model-00036-of-00206.safetensors",
+ "model.layers.3.mlps.1.up_proj.weight": "model-00036-of-00206.safetensors",
+ "model.layers.3.post_attention_layernorm.0.weight": "model-00036-of-00206.safetensors",
+ "model.layers.3.post_attention_layernorm.1.weight": "model-00036-of-00206.safetensors",
+ "model.layers.3.self_attn.0.embed_q.biases": "model-00036-of-00206.safetensors",
+ "model.layers.3.self_attn.0.embed_q.scales": "model-00036-of-00206.safetensors",
+ "model.layers.3.self_attn.0.embed_q.weight": "model-00036-of-00206.safetensors",
+ "model.layers.3.self_attn.0.indexer.k_norm.weight": "model-00036-of-00206.safetensors",
+ "model.layers.3.self_attn.0.indexer.weights_proj.biases": "model-00036-of-00206.safetensors",
+ "model.layers.3.self_attn.0.indexer.weights_proj.scales": "model-00036-of-00206.safetensors",
+ "model.layers.3.self_attn.0.indexer.weights_proj.weight": "model-00036-of-00206.safetensors",
+ "model.layers.3.self_attn.0.indexer.wk.biases": "model-00036-of-00206.safetensors",
+ "model.layers.3.self_attn.0.indexer.wk.scales": "model-00036-of-00206.safetensors",
+ "model.layers.3.self_attn.0.indexer.wk.weight": "model-00036-of-00206.safetensors",
+ "model.layers.3.self_attn.0.indexer.wq_b.biases": "model-00036-of-00206.safetensors",
+ "model.layers.3.self_attn.0.indexer.wq_b.scales": "model-00036-of-00206.safetensors",
+ "model.layers.3.self_attn.0.indexer.wq_b.weight": "model-00036-of-00206.safetensors",
+ "model.layers.3.self_attn.0.kv_a_layernorm.weight": "model-00036-of-00206.safetensors",
+ "model.layers.3.self_attn.0.kv_a_proj_with_mqa.biases": "model-00036-of-00206.safetensors",
+ "model.layers.3.self_attn.0.kv_a_proj_with_mqa.scales": "model-00036-of-00206.safetensors",
+ "model.layers.3.self_attn.0.kv_a_proj_with_mqa.weight": "model-00036-of-00206.safetensors",
+ "model.layers.3.self_attn.0.o_proj.biases": "model-00036-of-00206.safetensors",
+ "model.layers.3.self_attn.0.o_proj.scales": "model-00036-of-00206.safetensors",
+ "model.layers.3.self_attn.0.o_proj.weight": "model-00036-of-00206.safetensors",
+ "model.layers.3.self_attn.0.q_a_layernorm.weight": "model-00036-of-00206.safetensors",
+ "model.layers.3.self_attn.0.q_a_proj.biases": "model-00036-of-00206.safetensors",
+ "model.layers.3.self_attn.0.q_a_proj.scales": "model-00036-of-00206.safetensors",
+ "model.layers.3.self_attn.0.q_a_proj.weight": "model-00036-of-00206.safetensors",
+ "model.layers.3.self_attn.0.q_b_proj.biases": "model-00036-of-00206.safetensors",
+ "model.layers.3.self_attn.0.q_b_proj.scales": "model-00036-of-00206.safetensors",
+ "model.layers.3.self_attn.0.q_b_proj.weight": "model-00036-of-00206.safetensors",
+ "model.layers.3.self_attn.0.unembed_out.biases": "model-00036-of-00206.safetensors",
+ "model.layers.3.self_attn.0.unembed_out.scales": "model-00036-of-00206.safetensors",
+ "model.layers.3.self_attn.0.unembed_out.weight": "model-00036-of-00206.safetensors",
+ "model.layers.3.self_attn.1.embed_q.biases": "model-00036-of-00206.safetensors",
+ "model.layers.3.self_attn.1.embed_q.scales": "model-00036-of-00206.safetensors",
+ "model.layers.3.self_attn.1.embed_q.weight": "model-00036-of-00206.safetensors",
+ "model.layers.3.self_attn.1.kv_a_layernorm.weight": "model-00036-of-00206.safetensors",
+ "model.layers.3.self_attn.1.kv_a_proj_with_mqa.biases": "model-00036-of-00206.safetensors",
+ "model.layers.3.self_attn.1.kv_a_proj_with_mqa.scales": "model-00036-of-00206.safetensors",
+ "model.layers.3.self_attn.1.kv_a_proj_with_mqa.weight": "model-00036-of-00206.safetensors",
+ "model.layers.3.self_attn.1.o_proj.biases": "model-00036-of-00206.safetensors",
+ "model.layers.3.self_attn.1.o_proj.scales": "model-00036-of-00206.safetensors",
+ "model.layers.3.self_attn.1.o_proj.weight": "model-00036-of-00206.safetensors",
+ "model.layers.3.self_attn.1.q_a_layernorm.weight": "model-00036-of-00206.safetensors",
+ "model.layers.3.self_attn.1.q_a_proj.biases": "model-00036-of-00206.safetensors",
+ "model.layers.3.self_attn.1.q_a_proj.scales": "model-00036-of-00206.safetensors",
+ "model.layers.3.self_attn.1.q_a_proj.weight": "model-00036-of-00206.safetensors",
+ "model.layers.3.self_attn.1.q_b_proj.biases": "model-00036-of-00206.safetensors",
+ "model.layers.3.self_attn.1.q_b_proj.scales": "model-00036-of-00206.safetensors",
+ "model.layers.3.self_attn.1.q_b_proj.weight": "model-00036-of-00206.safetensors",
+ "model.layers.3.self_attn.1.unembed_out.biases": "model-00036-of-00206.safetensors",
+ "model.layers.3.self_attn.1.unembed_out.scales": "model-00036-of-00206.safetensors",
+ "model.layers.3.self_attn.1.unembed_out.weight": "model-00036-of-00206.safetensors",
+ "model.layers.30.input_layernorm.0.weight": "model-00171-of-00206.safetensors",
+ "model.layers.30.input_layernorm.1.weight": "model-00171-of-00206.safetensors",
+ "model.layers.30.mlp.router.classifier.biases": "model-00171-of-00206.safetensors",
+ "model.layers.30.mlp.router.classifier.scales": "model-00171-of-00206.safetensors",
+ "model.layers.30.mlp.router.classifier.weight": "model-00171-of-00206.safetensors",
+ "model.layers.30.mlp.router.e_score_correction_bias": "model-00171-of-00206.safetensors",
+ "model.layers.30.mlp.switch_mlp.down_proj.biases": "model-00171-of-00206.safetensors",
+ "model.layers.30.mlp.switch_mlp.down_proj.scales": "model-00170-of-00206.safetensors",
+ "model.layers.30.mlp.switch_mlp.down_proj.weight": "model-00170-of-00206.safetensors",
+ "model.layers.30.mlp.switch_mlp.gate_proj.biases": "model-00168-of-00206.safetensors",
+ "model.layers.30.mlp.switch_mlp.gate_proj.scales": "model-00167-of-00206.safetensors",
+ "model.layers.30.mlp.switch_mlp.gate_proj.weight": "model-00167-of-00206.safetensors",
+ "model.layers.30.mlp.switch_mlp.up_proj.biases": "model-00169-of-00206.safetensors",
+ "model.layers.30.mlp.switch_mlp.up_proj.scales": "model-00169-of-00206.safetensors",
+ "model.layers.30.mlp.switch_mlp.up_proj.weight": "model-00168-of-00206.safetensors",
+ "model.layers.30.mlps.0.down_proj.biases": "model-00171-of-00206.safetensors",
+ "model.layers.30.mlps.0.down_proj.scales": "model-00171-of-00206.safetensors",
+ "model.layers.30.mlps.0.down_proj.weight": "model-00171-of-00206.safetensors",
+ "model.layers.30.mlps.0.gate_proj.biases": "model-00171-of-00206.safetensors",
+ "model.layers.30.mlps.0.gate_proj.scales": "model-00171-of-00206.safetensors",
+ "model.layers.30.mlps.0.gate_proj.weight": "model-00171-of-00206.safetensors",
+ "model.layers.30.mlps.0.up_proj.biases": "model-00171-of-00206.safetensors",
+ "model.layers.30.mlps.0.up_proj.scales": "model-00171-of-00206.safetensors",
+ "model.layers.30.mlps.0.up_proj.weight": "model-00171-of-00206.safetensors",
+ "model.layers.30.mlps.1.down_proj.biases": "model-00171-of-00206.safetensors",
+ "model.layers.30.mlps.1.down_proj.scales": "model-00171-of-00206.safetensors",
+ "model.layers.30.mlps.1.down_proj.weight": "model-00171-of-00206.safetensors",
+ "model.layers.30.mlps.1.gate_proj.biases": "model-00171-of-00206.safetensors",
+ "model.layers.30.mlps.1.gate_proj.scales": "model-00171-of-00206.safetensors",
+ "model.layers.30.mlps.1.gate_proj.weight": "model-00171-of-00206.safetensors",
+ "model.layers.30.mlps.1.up_proj.biases": "model-00171-of-00206.safetensors",
+ "model.layers.30.mlps.1.up_proj.scales": "model-00171-of-00206.safetensors",
+ "model.layers.30.mlps.1.up_proj.weight": "model-00171-of-00206.safetensors",
+ "model.layers.30.post_attention_layernorm.0.weight": "model-00171-of-00206.safetensors",
+ "model.layers.30.post_attention_layernorm.1.weight": "model-00171-of-00206.safetensors",
+ "model.layers.30.self_attn.0.embed_q.biases": "model-00171-of-00206.safetensors",
+ "model.layers.30.self_attn.0.embed_q.scales": "model-00171-of-00206.safetensors",
+ "model.layers.30.self_attn.0.embed_q.weight": "model-00171-of-00206.safetensors",
+ "model.layers.30.self_attn.0.indexer.k_norm.weight": "model-00171-of-00206.safetensors",
+ "model.layers.30.self_attn.0.indexer.weights_proj.biases": "model-00171-of-00206.safetensors",
+ "model.layers.30.self_attn.0.indexer.weights_proj.scales": "model-00171-of-00206.safetensors",
+ "model.layers.30.self_attn.0.indexer.weights_proj.weight": "model-00171-of-00206.safetensors",
+ "model.layers.30.self_attn.0.indexer.wk.biases": "model-00171-of-00206.safetensors",
+ "model.layers.30.self_attn.0.indexer.wk.scales": "model-00171-of-00206.safetensors",
+ "model.layers.30.self_attn.0.indexer.wk.weight": "model-00171-of-00206.safetensors",
+ "model.layers.30.self_attn.0.indexer.wq_b.biases": "model-00171-of-00206.safetensors",
+ "model.layers.30.self_attn.0.indexer.wq_b.scales": "model-00171-of-00206.safetensors",
+ "model.layers.30.self_attn.0.indexer.wq_b.weight": "model-00171-of-00206.safetensors",
+ "model.layers.30.self_attn.0.kv_a_layernorm.weight": "model-00171-of-00206.safetensors",
+ "model.layers.30.self_attn.0.kv_a_proj_with_mqa.biases": "model-00171-of-00206.safetensors",
+ "model.layers.30.self_attn.0.kv_a_proj_with_mqa.scales": "model-00171-of-00206.safetensors",
+ "model.layers.30.self_attn.0.kv_a_proj_with_mqa.weight": "model-00171-of-00206.safetensors",
+ "model.layers.30.self_attn.0.o_proj.biases": "model-00171-of-00206.safetensors",
+ "model.layers.30.self_attn.0.o_proj.scales": "model-00171-of-00206.safetensors",
+ "model.layers.30.self_attn.0.o_proj.weight": "model-00171-of-00206.safetensors",
+ "model.layers.30.self_attn.0.q_a_layernorm.weight": "model-00171-of-00206.safetensors",
+ "model.layers.30.self_attn.0.q_a_proj.biases": "model-00171-of-00206.safetensors",
+ "model.layers.30.self_attn.0.q_a_proj.scales": "model-00171-of-00206.safetensors",
+ "model.layers.30.self_attn.0.q_a_proj.weight": "model-00171-of-00206.safetensors",
+ "model.layers.30.self_attn.0.q_b_proj.biases": "model-00171-of-00206.safetensors",
+ "model.layers.30.self_attn.0.q_b_proj.scales": "model-00171-of-00206.safetensors",
+ "model.layers.30.self_attn.0.q_b_proj.weight": "model-00171-of-00206.safetensors",
+ "model.layers.30.self_attn.0.unembed_out.biases": "model-00171-of-00206.safetensors",
+ "model.layers.30.self_attn.0.unembed_out.scales": "model-00171-of-00206.safetensors",
+ "model.layers.30.self_attn.0.unembed_out.weight": "model-00171-of-00206.safetensors",
+ "model.layers.30.self_attn.1.embed_q.biases": "model-00171-of-00206.safetensors",
+ "model.layers.30.self_attn.1.embed_q.scales": "model-00171-of-00206.safetensors",
+ "model.layers.30.self_attn.1.embed_q.weight": "model-00171-of-00206.safetensors",
+ "model.layers.30.self_attn.1.kv_a_layernorm.weight": "model-00171-of-00206.safetensors",
+ "model.layers.30.self_attn.1.kv_a_proj_with_mqa.biases": "model-00171-of-00206.safetensors",
+ "model.layers.30.self_attn.1.kv_a_proj_with_mqa.scales": "model-00171-of-00206.safetensors",
+ "model.layers.30.self_attn.1.kv_a_proj_with_mqa.weight": "model-00171-of-00206.safetensors",
+ "model.layers.30.self_attn.1.o_proj.biases": "model-00171-of-00206.safetensors",
+ "model.layers.30.self_attn.1.o_proj.scales": "model-00171-of-00206.safetensors",
+ "model.layers.30.self_attn.1.o_proj.weight": "model-00171-of-00206.safetensors",
+ "model.layers.30.self_attn.1.q_a_layernorm.weight": "model-00171-of-00206.safetensors",
+ "model.layers.30.self_attn.1.q_a_proj.biases": "model-00171-of-00206.safetensors",
+ "model.layers.30.self_attn.1.q_a_proj.scales": "model-00171-of-00206.safetensors",
+ "model.layers.30.self_attn.1.q_a_proj.weight": "model-00171-of-00206.safetensors",
+ "model.layers.30.self_attn.1.q_b_proj.biases": "model-00171-of-00206.safetensors",
+ "model.layers.30.self_attn.1.q_b_proj.scales": "model-00171-of-00206.safetensors",
+ "model.layers.30.self_attn.1.q_b_proj.weight": "model-00171-of-00206.safetensors",
+ "model.layers.30.self_attn.1.unembed_out.biases": "model-00171-of-00206.safetensors",
+ "model.layers.30.self_attn.1.unembed_out.scales": "model-00171-of-00206.safetensors",
+ "model.layers.30.self_attn.1.unembed_out.weight": "model-00171-of-00206.safetensors",
+ "model.layers.31.input_layernorm.0.weight": "model-00176-of-00206.safetensors",
+ "model.layers.31.input_layernorm.1.weight": "model-00176-of-00206.safetensors",
+ "model.layers.31.mlp.router.classifier.biases": "model-00176-of-00206.safetensors",
+ "model.layers.31.mlp.router.classifier.scales": "model-00176-of-00206.safetensors",
+ "model.layers.31.mlp.router.classifier.weight": "model-00176-of-00206.safetensors",
+ "model.layers.31.mlp.router.e_score_correction_bias": "model-00176-of-00206.safetensors",
+ "model.layers.31.mlp.switch_mlp.down_proj.biases": "model-00176-of-00206.safetensors",
+ "model.layers.31.mlp.switch_mlp.down_proj.scales": "model-00175-of-00206.safetensors",
+ "model.layers.31.mlp.switch_mlp.down_proj.weight": "model-00175-of-00206.safetensors",
+ "model.layers.31.mlp.switch_mlp.gate_proj.biases": "model-00173-of-00206.safetensors",
+ "model.layers.31.mlp.switch_mlp.gate_proj.scales": "model-00172-of-00206.safetensors",
+ "model.layers.31.mlp.switch_mlp.gate_proj.weight": "model-00172-of-00206.safetensors",
+ "model.layers.31.mlp.switch_mlp.up_proj.biases": "model-00174-of-00206.safetensors",
+ "model.layers.31.mlp.switch_mlp.up_proj.scales": "model-00174-of-00206.safetensors",
+ "model.layers.31.mlp.switch_mlp.up_proj.weight": "model-00173-of-00206.safetensors",
+ "model.layers.31.mlps.0.down_proj.biases": "model-00176-of-00206.safetensors",
+ "model.layers.31.mlps.0.down_proj.scales": "model-00176-of-00206.safetensors",
+ "model.layers.31.mlps.0.down_proj.weight": "model-00176-of-00206.safetensors",
+ "model.layers.31.mlps.0.gate_proj.biases": "model-00176-of-00206.safetensors",
+ "model.layers.31.mlps.0.gate_proj.scales": "model-00176-of-00206.safetensors",
+ "model.layers.31.mlps.0.gate_proj.weight": "model-00176-of-00206.safetensors",
+ "model.layers.31.mlps.0.up_proj.biases": "model-00176-of-00206.safetensors",
+ "model.layers.31.mlps.0.up_proj.scales": "model-00176-of-00206.safetensors",
+ "model.layers.31.mlps.0.up_proj.weight": "model-00176-of-00206.safetensors",
+ "model.layers.31.mlps.1.down_proj.biases": "model-00176-of-00206.safetensors",
+ "model.layers.31.mlps.1.down_proj.scales": "model-00176-of-00206.safetensors",
+ "model.layers.31.mlps.1.down_proj.weight": "model-00176-of-00206.safetensors",
+ "model.layers.31.mlps.1.gate_proj.biases": "model-00176-of-00206.safetensors",
+ "model.layers.31.mlps.1.gate_proj.scales": "model-00176-of-00206.safetensors",
+ "model.layers.31.mlps.1.gate_proj.weight": "model-00176-of-00206.safetensors",
+ "model.layers.31.mlps.1.up_proj.biases": "model-00176-of-00206.safetensors",
+ "model.layers.31.mlps.1.up_proj.scales": "model-00176-of-00206.safetensors",
+ "model.layers.31.mlps.1.up_proj.weight": "model-00176-of-00206.safetensors",
+ "model.layers.31.post_attention_layernorm.0.weight": "model-00176-of-00206.safetensors",
+ "model.layers.31.post_attention_layernorm.1.weight": "model-00176-of-00206.safetensors",
+ "model.layers.31.self_attn.0.embed_q.biases": "model-00176-of-00206.safetensors",
+ "model.layers.31.self_attn.0.embed_q.scales": "model-00176-of-00206.safetensors",
+ "model.layers.31.self_attn.0.embed_q.weight": "model-00176-of-00206.safetensors",
+ "model.layers.31.self_attn.0.indexer.k_norm.weight": "model-00176-of-00206.safetensors",
+ "model.layers.31.self_attn.0.indexer.weights_proj.biases": "model-00176-of-00206.safetensors",
+ "model.layers.31.self_attn.0.indexer.weights_proj.scales": "model-00176-of-00206.safetensors",
+ "model.layers.31.self_attn.0.indexer.weights_proj.weight": "model-00176-of-00206.safetensors",
+ "model.layers.31.self_attn.0.indexer.wk.biases": "model-00176-of-00206.safetensors",
+ "model.layers.31.self_attn.0.indexer.wk.scales": "model-00176-of-00206.safetensors",
+ "model.layers.31.self_attn.0.indexer.wk.weight": "model-00176-of-00206.safetensors",
+ "model.layers.31.self_attn.0.indexer.wq_b.biases": "model-00176-of-00206.safetensors",
+ "model.layers.31.self_attn.0.indexer.wq_b.scales": "model-00176-of-00206.safetensors",
+ "model.layers.31.self_attn.0.indexer.wq_b.weight": "model-00176-of-00206.safetensors",
+ "model.layers.31.self_attn.0.kv_a_layernorm.weight": "model-00176-of-00206.safetensors",
+ "model.layers.31.self_attn.0.kv_a_proj_with_mqa.biases": "model-00176-of-00206.safetensors",
+ "model.layers.31.self_attn.0.kv_a_proj_with_mqa.scales": "model-00176-of-00206.safetensors",
+ "model.layers.31.self_attn.0.kv_a_proj_with_mqa.weight": "model-00176-of-00206.safetensors",
+ "model.layers.31.self_attn.0.o_proj.biases": "model-00176-of-00206.safetensors",
+ "model.layers.31.self_attn.0.o_proj.scales": "model-00176-of-00206.safetensors",
+ "model.layers.31.self_attn.0.o_proj.weight": "model-00176-of-00206.safetensors",
+ "model.layers.31.self_attn.0.q_a_layernorm.weight": "model-00176-of-00206.safetensors",
+ "model.layers.31.self_attn.0.q_a_proj.biases": "model-00176-of-00206.safetensors",
+ "model.layers.31.self_attn.0.q_a_proj.scales": "model-00176-of-00206.safetensors",
+ "model.layers.31.self_attn.0.q_a_proj.weight": "model-00176-of-00206.safetensors",
+ "model.layers.31.self_attn.0.q_b_proj.biases": "model-00176-of-00206.safetensors",
+ "model.layers.31.self_attn.0.q_b_proj.scales": "model-00176-of-00206.safetensors",
+ "model.layers.31.self_attn.0.q_b_proj.weight": "model-00176-of-00206.safetensors",
+ "model.layers.31.self_attn.0.unembed_out.biases": "model-00176-of-00206.safetensors",
+ "model.layers.31.self_attn.0.unembed_out.scales": "model-00176-of-00206.safetensors",
+ "model.layers.31.self_attn.0.unembed_out.weight": "model-00176-of-00206.safetensors",
+ "model.layers.31.self_attn.1.embed_q.biases": "model-00176-of-00206.safetensors",
+ "model.layers.31.self_attn.1.embed_q.scales": "model-00176-of-00206.safetensors",
+ "model.layers.31.self_attn.1.embed_q.weight": "model-00176-of-00206.safetensors",
+ "model.layers.31.self_attn.1.kv_a_layernorm.weight": "model-00176-of-00206.safetensors",
+ "model.layers.31.self_attn.1.kv_a_proj_with_mqa.biases": "model-00176-of-00206.safetensors",
+ "model.layers.31.self_attn.1.kv_a_proj_with_mqa.scales": "model-00176-of-00206.safetensors",
+ "model.layers.31.self_attn.1.kv_a_proj_with_mqa.weight": "model-00176-of-00206.safetensors",
+ "model.layers.31.self_attn.1.o_proj.biases": "model-00176-of-00206.safetensors",
+ "model.layers.31.self_attn.1.o_proj.scales": "model-00176-of-00206.safetensors",
+ "model.layers.31.self_attn.1.o_proj.weight": "model-00176-of-00206.safetensors",
+ "model.layers.31.self_attn.1.q_a_layernorm.weight": "model-00176-of-00206.safetensors",
+ "model.layers.31.self_attn.1.q_a_proj.biases": "model-00176-of-00206.safetensors",
+ "model.layers.31.self_attn.1.q_a_proj.scales": "model-00176-of-00206.safetensors",
+ "model.layers.31.self_attn.1.q_a_proj.weight": "model-00176-of-00206.safetensors",
+ "model.layers.31.self_attn.1.q_b_proj.biases": "model-00176-of-00206.safetensors",
+ "model.layers.31.self_attn.1.q_b_proj.scales": "model-00176-of-00206.safetensors",
+ "model.layers.31.self_attn.1.q_b_proj.weight": "model-00176-of-00206.safetensors",
+ "model.layers.31.self_attn.1.unembed_out.biases": "model-00176-of-00206.safetensors",
+ "model.layers.31.self_attn.1.unembed_out.scales": "model-00176-of-00206.safetensors",
+ "model.layers.31.self_attn.1.unembed_out.weight": "model-00176-of-00206.safetensors",
+ "model.layers.32.input_layernorm.0.weight": "model-00181-of-00206.safetensors",
+ "model.layers.32.input_layernorm.1.weight": "model-00181-of-00206.safetensors",
+ "model.layers.32.mlp.router.classifier.biases": "model-00181-of-00206.safetensors",
+ "model.layers.32.mlp.router.classifier.scales": "model-00181-of-00206.safetensors",
+ "model.layers.32.mlp.router.classifier.weight": "model-00181-of-00206.safetensors",
+ "model.layers.32.mlp.router.e_score_correction_bias": "model-00181-of-00206.safetensors",
+ "model.layers.32.mlp.switch_mlp.down_proj.biases": "model-00181-of-00206.safetensors",
+ "model.layers.32.mlp.switch_mlp.down_proj.scales": "model-00180-of-00206.safetensors",
+ "model.layers.32.mlp.switch_mlp.down_proj.weight": "model-00180-of-00206.safetensors",
+ "model.layers.32.mlp.switch_mlp.gate_proj.biases": "model-00178-of-00206.safetensors",
+ "model.layers.32.mlp.switch_mlp.gate_proj.scales": "model-00177-of-00206.safetensors",
+ "model.layers.32.mlp.switch_mlp.gate_proj.weight": "model-00177-of-00206.safetensors",
+ "model.layers.32.mlp.switch_mlp.up_proj.biases": "model-00179-of-00206.safetensors",
+ "model.layers.32.mlp.switch_mlp.up_proj.scales": "model-00179-of-00206.safetensors",
+ "model.layers.32.mlp.switch_mlp.up_proj.weight": "model-00178-of-00206.safetensors",
+ "model.layers.32.mlps.0.down_proj.biases": "model-00181-of-00206.safetensors",
+ "model.layers.32.mlps.0.down_proj.scales": "model-00181-of-00206.safetensors",
+ "model.layers.32.mlps.0.down_proj.weight": "model-00181-of-00206.safetensors",
+ "model.layers.32.mlps.0.gate_proj.biases": "model-00181-of-00206.safetensors",
+ "model.layers.32.mlps.0.gate_proj.scales": "model-00181-of-00206.safetensors",
+ "model.layers.32.mlps.0.gate_proj.weight": "model-00181-of-00206.safetensors",
+ "model.layers.32.mlps.0.up_proj.biases": "model-00181-of-00206.safetensors",
+ "model.layers.32.mlps.0.up_proj.scales": "model-00181-of-00206.safetensors",
+ "model.layers.32.mlps.0.up_proj.weight": "model-00181-of-00206.safetensors",
+ "model.layers.32.mlps.1.down_proj.biases": "model-00181-of-00206.safetensors",
+ "model.layers.32.mlps.1.down_proj.scales": "model-00181-of-00206.safetensors",
+ "model.layers.32.mlps.1.down_proj.weight": "model-00181-of-00206.safetensors",
+ "model.layers.32.mlps.1.gate_proj.biases": "model-00181-of-00206.safetensors",
+ "model.layers.32.mlps.1.gate_proj.scales": "model-00181-of-00206.safetensors",
+ "model.layers.32.mlps.1.gate_proj.weight": "model-00181-of-00206.safetensors",
+ "model.layers.32.mlps.1.up_proj.biases": "model-00181-of-00206.safetensors",
+ "model.layers.32.mlps.1.up_proj.scales": "model-00181-of-00206.safetensors",
+ "model.layers.32.mlps.1.up_proj.weight": "model-00181-of-00206.safetensors",
+ "model.layers.32.post_attention_layernorm.0.weight": "model-00181-of-00206.safetensors",
+ "model.layers.32.post_attention_layernorm.1.weight": "model-00181-of-00206.safetensors",
+ "model.layers.32.self_attn.0.embed_q.biases": "model-00181-of-00206.safetensors",
+ "model.layers.32.self_attn.0.embed_q.scales": "model-00181-of-00206.safetensors",
+ "model.layers.32.self_attn.0.embed_q.weight": "model-00181-of-00206.safetensors",
+ "model.layers.32.self_attn.0.indexer.k_norm.weight": "model-00181-of-00206.safetensors",
+ "model.layers.32.self_attn.0.indexer.weights_proj.biases": "model-00181-of-00206.safetensors",
+ "model.layers.32.self_attn.0.indexer.weights_proj.scales": "model-00181-of-00206.safetensors",
+ "model.layers.32.self_attn.0.indexer.weights_proj.weight": "model-00181-of-00206.safetensors",
+ "model.layers.32.self_attn.0.indexer.wk.biases": "model-00181-of-00206.safetensors",
+ "model.layers.32.self_attn.0.indexer.wk.scales": "model-00181-of-00206.safetensors",
+ "model.layers.32.self_attn.0.indexer.wk.weight": "model-00181-of-00206.safetensors",
+ "model.layers.32.self_attn.0.indexer.wq_b.biases": "model-00181-of-00206.safetensors",
+ "model.layers.32.self_attn.0.indexer.wq_b.scales": "model-00181-of-00206.safetensors",
+ "model.layers.32.self_attn.0.indexer.wq_b.weight": "model-00181-of-00206.safetensors",
+ "model.layers.32.self_attn.0.kv_a_layernorm.weight": "model-00181-of-00206.safetensors",
+ "model.layers.32.self_attn.0.kv_a_proj_with_mqa.biases": "model-00181-of-00206.safetensors",
+ "model.layers.32.self_attn.0.kv_a_proj_with_mqa.scales": "model-00181-of-00206.safetensors",
+ "model.layers.32.self_attn.0.kv_a_proj_with_mqa.weight": "model-00181-of-00206.safetensors",
+ "model.layers.32.self_attn.0.o_proj.biases": "model-00181-of-00206.safetensors",
+ "model.layers.32.self_attn.0.o_proj.scales": "model-00181-of-00206.safetensors",
+ "model.layers.32.self_attn.0.o_proj.weight": "model-00181-of-00206.safetensors",
+ "model.layers.32.self_attn.0.q_a_layernorm.weight": "model-00181-of-00206.safetensors",
+ "model.layers.32.self_attn.0.q_a_proj.biases": "model-00181-of-00206.safetensors",
+ "model.layers.32.self_attn.0.q_a_proj.scales": "model-00181-of-00206.safetensors",
+ "model.layers.32.self_attn.0.q_a_proj.weight": "model-00181-of-00206.safetensors",
+ "model.layers.32.self_attn.0.q_b_proj.biases": "model-00181-of-00206.safetensors",
+ "model.layers.32.self_attn.0.q_b_proj.scales": "model-00181-of-00206.safetensors",
+ "model.layers.32.self_attn.0.q_b_proj.weight": "model-00181-of-00206.safetensors",
+ "model.layers.32.self_attn.0.unembed_out.biases": "model-00181-of-00206.safetensors",
+ "model.layers.32.self_attn.0.unembed_out.scales": "model-00181-of-00206.safetensors",
+ "model.layers.32.self_attn.0.unembed_out.weight": "model-00181-of-00206.safetensors",
+ "model.layers.32.self_attn.1.embed_q.biases": "model-00181-of-00206.safetensors",
+ "model.layers.32.self_attn.1.embed_q.scales": "model-00181-of-00206.safetensors",
+ "model.layers.32.self_attn.1.embed_q.weight": "model-00181-of-00206.safetensors",
+ "model.layers.32.self_attn.1.kv_a_layernorm.weight": "model-00181-of-00206.safetensors",
+ "model.layers.32.self_attn.1.kv_a_proj_with_mqa.biases": "model-00181-of-00206.safetensors",
+ "model.layers.32.self_attn.1.kv_a_proj_with_mqa.scales": "model-00181-of-00206.safetensors",
+ "model.layers.32.self_attn.1.kv_a_proj_with_mqa.weight": "model-00181-of-00206.safetensors",
+ "model.layers.32.self_attn.1.o_proj.biases": "model-00181-of-00206.safetensors",
+ "model.layers.32.self_attn.1.o_proj.scales": "model-00181-of-00206.safetensors",
+ "model.layers.32.self_attn.1.o_proj.weight": "model-00181-of-00206.safetensors",
+ "model.layers.32.self_attn.1.q_a_layernorm.weight": "model-00181-of-00206.safetensors",
+ "model.layers.32.self_attn.1.q_a_proj.biases": "model-00181-of-00206.safetensors",
+ "model.layers.32.self_attn.1.q_a_proj.scales": "model-00181-of-00206.safetensors",
+ "model.layers.32.self_attn.1.q_a_proj.weight": "model-00181-of-00206.safetensors",
+ "model.layers.32.self_attn.1.q_b_proj.biases": "model-00181-of-00206.safetensors",
+ "model.layers.32.self_attn.1.q_b_proj.scales": "model-00181-of-00206.safetensors",
+ "model.layers.32.self_attn.1.q_b_proj.weight": "model-00181-of-00206.safetensors",
+ "model.layers.32.self_attn.1.unembed_out.biases": "model-00181-of-00206.safetensors",
+ "model.layers.32.self_attn.1.unembed_out.scales": "model-00181-of-00206.safetensors",
+ "model.layers.32.self_attn.1.unembed_out.weight": "model-00181-of-00206.safetensors",
+ "model.layers.33.input_layernorm.0.weight": "model-00186-of-00206.safetensors",
+ "model.layers.33.input_layernorm.1.weight": "model-00186-of-00206.safetensors",
+ "model.layers.33.mlp.router.classifier.biases": "model-00186-of-00206.safetensors",
+ "model.layers.33.mlp.router.classifier.scales": "model-00186-of-00206.safetensors",
+ "model.layers.33.mlp.router.classifier.weight": "model-00186-of-00206.safetensors",
+ "model.layers.33.mlp.router.e_score_correction_bias": "model-00186-of-00206.safetensors",
+ "model.layers.33.mlp.switch_mlp.down_proj.biases": "model-00186-of-00206.safetensors",
+ "model.layers.33.mlp.switch_mlp.down_proj.scales": "model-00185-of-00206.safetensors",
+ "model.layers.33.mlp.switch_mlp.down_proj.weight": "model-00185-of-00206.safetensors",
+ "model.layers.33.mlp.switch_mlp.gate_proj.biases": "model-00183-of-00206.safetensors",
+ "model.layers.33.mlp.switch_mlp.gate_proj.scales": "model-00182-of-00206.safetensors",
+ "model.layers.33.mlp.switch_mlp.gate_proj.weight": "model-00182-of-00206.safetensors",
+ "model.layers.33.mlp.switch_mlp.up_proj.biases": "model-00184-of-00206.safetensors",
+ "model.layers.33.mlp.switch_mlp.up_proj.scales": "model-00184-of-00206.safetensors",
+ "model.layers.33.mlp.switch_mlp.up_proj.weight": "model-00183-of-00206.safetensors",
+ "model.layers.33.mlps.0.down_proj.biases": "model-00186-of-00206.safetensors",
+ "model.layers.33.mlps.0.down_proj.scales": "model-00186-of-00206.safetensors",
+ "model.layers.33.mlps.0.down_proj.weight": "model-00186-of-00206.safetensors",
+ "model.layers.33.mlps.0.gate_proj.biases": "model-00186-of-00206.safetensors",
+ "model.layers.33.mlps.0.gate_proj.scales": "model-00186-of-00206.safetensors",
+ "model.layers.33.mlps.0.gate_proj.weight": "model-00186-of-00206.safetensors",
+ "model.layers.33.mlps.0.up_proj.biases": "model-00186-of-00206.safetensors",
+ "model.layers.33.mlps.0.up_proj.scales": "model-00186-of-00206.safetensors",
+ "model.layers.33.mlps.0.up_proj.weight": "model-00186-of-00206.safetensors",
+ "model.layers.33.mlps.1.down_proj.biases": "model-00186-of-00206.safetensors",
+ "model.layers.33.mlps.1.down_proj.scales": "model-00186-of-00206.safetensors",
+ "model.layers.33.mlps.1.down_proj.weight": "model-00186-of-00206.safetensors",
+ "model.layers.33.mlps.1.gate_proj.biases": "model-00186-of-00206.safetensors",
+ "model.layers.33.mlps.1.gate_proj.scales": "model-00186-of-00206.safetensors",
+ "model.layers.33.mlps.1.gate_proj.weight": "model-00186-of-00206.safetensors",
+ "model.layers.33.mlps.1.up_proj.biases": "model-00186-of-00206.safetensors",
+ "model.layers.33.mlps.1.up_proj.scales": "model-00186-of-00206.safetensors",
+ "model.layers.33.mlps.1.up_proj.weight": "model-00186-of-00206.safetensors",
+ "model.layers.33.post_attention_layernorm.0.weight": "model-00186-of-00206.safetensors",
+ "model.layers.33.post_attention_layernorm.1.weight": "model-00186-of-00206.safetensors",
+ "model.layers.33.self_attn.0.embed_q.biases": "model-00186-of-00206.safetensors",
+ "model.layers.33.self_attn.0.embed_q.scales": "model-00186-of-00206.safetensors",
+ "model.layers.33.self_attn.0.embed_q.weight": "model-00186-of-00206.safetensors",
+ "model.layers.33.self_attn.0.indexer.k_norm.weight": "model-00186-of-00206.safetensors",
+ "model.layers.33.self_attn.0.indexer.weights_proj.biases": "model-00186-of-00206.safetensors",
+ "model.layers.33.self_attn.0.indexer.weights_proj.scales": "model-00186-of-00206.safetensors",
+ "model.layers.33.self_attn.0.indexer.weights_proj.weight": "model-00186-of-00206.safetensors",
+ "model.layers.33.self_attn.0.indexer.wk.biases": "model-00186-of-00206.safetensors",
+ "model.layers.33.self_attn.0.indexer.wk.scales": "model-00186-of-00206.safetensors",
+ "model.layers.33.self_attn.0.indexer.wk.weight": "model-00186-of-00206.safetensors",
+ "model.layers.33.self_attn.0.indexer.wq_b.biases": "model-00186-of-00206.safetensors",
+ "model.layers.33.self_attn.0.indexer.wq_b.scales": "model-00186-of-00206.safetensors",
+ "model.layers.33.self_attn.0.indexer.wq_b.weight": "model-00186-of-00206.safetensors",
+ "model.layers.33.self_attn.0.kv_a_layernorm.weight": "model-00186-of-00206.safetensors",
+ "model.layers.33.self_attn.0.kv_a_proj_with_mqa.biases": "model-00186-of-00206.safetensors",
+ "model.layers.33.self_attn.0.kv_a_proj_with_mqa.scales": "model-00186-of-00206.safetensors",
+ "model.layers.33.self_attn.0.kv_a_proj_with_mqa.weight": "model-00186-of-00206.safetensors",
+ "model.layers.33.self_attn.0.o_proj.biases": "model-00186-of-00206.safetensors",
+ "model.layers.33.self_attn.0.o_proj.scales": "model-00186-of-00206.safetensors",
+ "model.layers.33.self_attn.0.o_proj.weight": "model-00186-of-00206.safetensors",
+ "model.layers.33.self_attn.0.q_a_layernorm.weight": "model-00186-of-00206.safetensors",
+ "model.layers.33.self_attn.0.q_a_proj.biases": "model-00186-of-00206.safetensors",
+ "model.layers.33.self_attn.0.q_a_proj.scales": "model-00186-of-00206.safetensors",
+ "model.layers.33.self_attn.0.q_a_proj.weight": "model-00186-of-00206.safetensors",
+ "model.layers.33.self_attn.0.q_b_proj.biases": "model-00186-of-00206.safetensors",
+ "model.layers.33.self_attn.0.q_b_proj.scales": "model-00186-of-00206.safetensors",
+ "model.layers.33.self_attn.0.q_b_proj.weight": "model-00186-of-00206.safetensors",
+ "model.layers.33.self_attn.0.unembed_out.biases": "model-00186-of-00206.safetensors",
+ "model.layers.33.self_attn.0.unembed_out.scales": "model-00186-of-00206.safetensors",
+ "model.layers.33.self_attn.0.unembed_out.weight": "model-00186-of-00206.safetensors",
+ "model.layers.33.self_attn.1.embed_q.biases": "model-00186-of-00206.safetensors",
+ "model.layers.33.self_attn.1.embed_q.scales": "model-00186-of-00206.safetensors",
+ "model.layers.33.self_attn.1.embed_q.weight": "model-00186-of-00206.safetensors",
+ "model.layers.33.self_attn.1.kv_a_layernorm.weight": "model-00186-of-00206.safetensors",
+ "model.layers.33.self_attn.1.kv_a_proj_with_mqa.biases": "model-00186-of-00206.safetensors",
+ "model.layers.33.self_attn.1.kv_a_proj_with_mqa.scales": "model-00186-of-00206.safetensors",
+ "model.layers.33.self_attn.1.kv_a_proj_with_mqa.weight": "model-00186-of-00206.safetensors",
+ "model.layers.33.self_attn.1.o_proj.biases": "model-00186-of-00206.safetensors",
+ "model.layers.33.self_attn.1.o_proj.scales": "model-00186-of-00206.safetensors",
+ "model.layers.33.self_attn.1.o_proj.weight": "model-00186-of-00206.safetensors",
+ "model.layers.33.self_attn.1.q_a_layernorm.weight": "model-00186-of-00206.safetensors",
+ "model.layers.33.self_attn.1.q_a_proj.biases": "model-00186-of-00206.safetensors",
+ "model.layers.33.self_attn.1.q_a_proj.scales": "model-00186-of-00206.safetensors",
+ "model.layers.33.self_attn.1.q_a_proj.weight": "model-00186-of-00206.safetensors",
+ "model.layers.33.self_attn.1.q_b_proj.biases": "model-00186-of-00206.safetensors",
+ "model.layers.33.self_attn.1.q_b_proj.scales": "model-00186-of-00206.safetensors",
+ "model.layers.33.self_attn.1.q_b_proj.weight": "model-00186-of-00206.safetensors",
+ "model.layers.33.self_attn.1.unembed_out.biases": "model-00186-of-00206.safetensors",
+ "model.layers.33.self_attn.1.unembed_out.scales": "model-00186-of-00206.safetensors",
+ "model.layers.33.self_attn.1.unembed_out.weight": "model-00186-of-00206.safetensors",
+ "model.layers.34.input_layernorm.0.weight": "model-00191-of-00206.safetensors",
+ "model.layers.34.input_layernorm.1.weight": "model-00191-of-00206.safetensors",
+ "model.layers.34.mlp.router.classifier.biases": "model-00191-of-00206.safetensors",
+ "model.layers.34.mlp.router.classifier.scales": "model-00191-of-00206.safetensors",
+ "model.layers.34.mlp.router.classifier.weight": "model-00191-of-00206.safetensors",
+ "model.layers.34.mlp.router.e_score_correction_bias": "model-00191-of-00206.safetensors",
+ "model.layers.34.mlp.switch_mlp.down_proj.biases": "model-00191-of-00206.safetensors",
+ "model.layers.34.mlp.switch_mlp.down_proj.scales": "model-00190-of-00206.safetensors",
+ "model.layers.34.mlp.switch_mlp.down_proj.weight": "model-00190-of-00206.safetensors",
+ "model.layers.34.mlp.switch_mlp.gate_proj.biases": "model-00188-of-00206.safetensors",
+ "model.layers.34.mlp.switch_mlp.gate_proj.scales": "model-00187-of-00206.safetensors",
+ "model.layers.34.mlp.switch_mlp.gate_proj.weight": "model-00187-of-00206.safetensors",
+ "model.layers.34.mlp.switch_mlp.up_proj.biases": "model-00189-of-00206.safetensors",
+ "model.layers.34.mlp.switch_mlp.up_proj.scales": "model-00189-of-00206.safetensors",
+ "model.layers.34.mlp.switch_mlp.up_proj.weight": "model-00188-of-00206.safetensors",
+ "model.layers.34.mlps.0.down_proj.biases": "model-00191-of-00206.safetensors",
+ "model.layers.34.mlps.0.down_proj.scales": "model-00191-of-00206.safetensors",
+ "model.layers.34.mlps.0.down_proj.weight": "model-00191-of-00206.safetensors",
+ "model.layers.34.mlps.0.gate_proj.biases": "model-00191-of-00206.safetensors",
+ "model.layers.34.mlps.0.gate_proj.scales": "model-00191-of-00206.safetensors",
+ "model.layers.34.mlps.0.gate_proj.weight": "model-00191-of-00206.safetensors",
+ "model.layers.34.mlps.0.up_proj.biases": "model-00191-of-00206.safetensors",
+ "model.layers.34.mlps.0.up_proj.scales": "model-00191-of-00206.safetensors",
+ "model.layers.34.mlps.0.up_proj.weight": "model-00191-of-00206.safetensors",
+ "model.layers.34.mlps.1.down_proj.biases": "model-00191-of-00206.safetensors",
+ "model.layers.34.mlps.1.down_proj.scales": "model-00191-of-00206.safetensors",
+ "model.layers.34.mlps.1.down_proj.weight": "model-00191-of-00206.safetensors",
+ "model.layers.34.mlps.1.gate_proj.biases": "model-00191-of-00206.safetensors",
+ "model.layers.34.mlps.1.gate_proj.scales": "model-00191-of-00206.safetensors",
+ "model.layers.34.mlps.1.gate_proj.weight": "model-00191-of-00206.safetensors",
+ "model.layers.34.mlps.1.up_proj.biases": "model-00191-of-00206.safetensors",
+ "model.layers.34.mlps.1.up_proj.scales": "model-00191-of-00206.safetensors",
+ "model.layers.34.mlps.1.up_proj.weight": "model-00191-of-00206.safetensors",
+ "model.layers.34.post_attention_layernorm.0.weight": "model-00191-of-00206.safetensors",
+ "model.layers.34.post_attention_layernorm.1.weight": "model-00191-of-00206.safetensors",
+ "model.layers.34.self_attn.0.embed_q.biases": "model-00191-of-00206.safetensors",
+ "model.layers.34.self_attn.0.embed_q.scales": "model-00191-of-00206.safetensors",
+ "model.layers.34.self_attn.0.embed_q.weight": "model-00191-of-00206.safetensors",
+ "model.layers.34.self_attn.0.indexer.k_norm.weight": "model-00191-of-00206.safetensors",
+ "model.layers.34.self_attn.0.indexer.weights_proj.biases": "model-00191-of-00206.safetensors",
+ "model.layers.34.self_attn.0.indexer.weights_proj.scales": "model-00191-of-00206.safetensors",
+ "model.layers.34.self_attn.0.indexer.weights_proj.weight": "model-00191-of-00206.safetensors",
+ "model.layers.34.self_attn.0.indexer.wk.biases": "model-00191-of-00206.safetensors",
+ "model.layers.34.self_attn.0.indexer.wk.scales": "model-00191-of-00206.safetensors",
+ "model.layers.34.self_attn.0.indexer.wk.weight": "model-00191-of-00206.safetensors",
+ "model.layers.34.self_attn.0.indexer.wq_b.biases": "model-00191-of-00206.safetensors",
+ "model.layers.34.self_attn.0.indexer.wq_b.scales": "model-00191-of-00206.safetensors",
+ "model.layers.34.self_attn.0.indexer.wq_b.weight": "model-00191-of-00206.safetensors",
+ "model.layers.34.self_attn.0.kv_a_layernorm.weight": "model-00191-of-00206.safetensors",
+ "model.layers.34.self_attn.0.kv_a_proj_with_mqa.biases": "model-00191-of-00206.safetensors",
+ "model.layers.34.self_attn.0.kv_a_proj_with_mqa.scales": "model-00191-of-00206.safetensors",
+ "model.layers.34.self_attn.0.kv_a_proj_with_mqa.weight": "model-00191-of-00206.safetensors",
+ "model.layers.34.self_attn.0.o_proj.biases": "model-00191-of-00206.safetensors",
+ "model.layers.34.self_attn.0.o_proj.scales": "model-00191-of-00206.safetensors",
+ "model.layers.34.self_attn.0.o_proj.weight": "model-00191-of-00206.safetensors",
+ "model.layers.34.self_attn.0.q_a_layernorm.weight": "model-00191-of-00206.safetensors",
+ "model.layers.34.self_attn.0.q_a_proj.biases": "model-00191-of-00206.safetensors",
+ "model.layers.34.self_attn.0.q_a_proj.scales": "model-00191-of-00206.safetensors",
+ "model.layers.34.self_attn.0.q_a_proj.weight": "model-00191-of-00206.safetensors",
+ "model.layers.34.self_attn.0.q_b_proj.biases": "model-00191-of-00206.safetensors",
+ "model.layers.34.self_attn.0.q_b_proj.scales": "model-00191-of-00206.safetensors",
+ "model.layers.34.self_attn.0.q_b_proj.weight": "model-00191-of-00206.safetensors",
+ "model.layers.34.self_attn.0.unembed_out.biases": "model-00191-of-00206.safetensors",
+ "model.layers.34.self_attn.0.unembed_out.scales": "model-00191-of-00206.safetensors",
+ "model.layers.34.self_attn.0.unembed_out.weight": "model-00191-of-00206.safetensors",
+ "model.layers.34.self_attn.1.embed_q.biases": "model-00191-of-00206.safetensors",
+ "model.layers.34.self_attn.1.embed_q.scales": "model-00191-of-00206.safetensors",
+ "model.layers.34.self_attn.1.embed_q.weight": "model-00191-of-00206.safetensors",
+ "model.layers.34.self_attn.1.kv_a_layernorm.weight": "model-00191-of-00206.safetensors",
+ "model.layers.34.self_attn.1.kv_a_proj_with_mqa.biases": "model-00191-of-00206.safetensors",
+ "model.layers.34.self_attn.1.kv_a_proj_with_mqa.scales": "model-00191-of-00206.safetensors",
+ "model.layers.34.self_attn.1.kv_a_proj_with_mqa.weight": "model-00191-of-00206.safetensors",
+ "model.layers.34.self_attn.1.o_proj.biases": "model-00191-of-00206.safetensors",
+ "model.layers.34.self_attn.1.o_proj.scales": "model-00191-of-00206.safetensors",
+ "model.layers.34.self_attn.1.o_proj.weight": "model-00191-of-00206.safetensors",
+ "model.layers.34.self_attn.1.q_a_layernorm.weight": "model-00191-of-00206.safetensors",
+ "model.layers.34.self_attn.1.q_a_proj.biases": "model-00191-of-00206.safetensors",
+ "model.layers.34.self_attn.1.q_a_proj.scales": "model-00191-of-00206.safetensors",
+ "model.layers.34.self_attn.1.q_a_proj.weight": "model-00191-of-00206.safetensors",
+ "model.layers.34.self_attn.1.q_b_proj.biases": "model-00191-of-00206.safetensors",
+ "model.layers.34.self_attn.1.q_b_proj.scales": "model-00191-of-00206.safetensors",
+ "model.layers.34.self_attn.1.q_b_proj.weight": "model-00191-of-00206.safetensors",
+ "model.layers.34.self_attn.1.unembed_out.biases": "model-00191-of-00206.safetensors",
+ "model.layers.34.self_attn.1.unembed_out.scales": "model-00191-of-00206.safetensors",
+ "model.layers.34.self_attn.1.unembed_out.weight": "model-00191-of-00206.safetensors",
+ "model.layers.35.input_layernorm.0.weight": "model-00196-of-00206.safetensors",
+ "model.layers.35.input_layernorm.1.weight": "model-00196-of-00206.safetensors",
+ "model.layers.35.mlp.router.classifier.biases": "model-00196-of-00206.safetensors",
+ "model.layers.35.mlp.router.classifier.scales": "model-00196-of-00206.safetensors",
+ "model.layers.35.mlp.router.classifier.weight": "model-00196-of-00206.safetensors",
+ "model.layers.35.mlp.router.e_score_correction_bias": "model-00196-of-00206.safetensors",
+ "model.layers.35.mlp.switch_mlp.down_proj.biases": "model-00196-of-00206.safetensors",
+ "model.layers.35.mlp.switch_mlp.down_proj.scales": "model-00195-of-00206.safetensors",
+ "model.layers.35.mlp.switch_mlp.down_proj.weight": "model-00195-of-00206.safetensors",
+ "model.layers.35.mlp.switch_mlp.gate_proj.biases": "model-00193-of-00206.safetensors",
+ "model.layers.35.mlp.switch_mlp.gate_proj.scales": "model-00192-of-00206.safetensors",
+ "model.layers.35.mlp.switch_mlp.gate_proj.weight": "model-00192-of-00206.safetensors",
+ "model.layers.35.mlp.switch_mlp.up_proj.biases": "model-00194-of-00206.safetensors",
+ "model.layers.35.mlp.switch_mlp.up_proj.scales": "model-00194-of-00206.safetensors",
+ "model.layers.35.mlp.switch_mlp.up_proj.weight": "model-00193-of-00206.safetensors",
+ "model.layers.35.mlps.0.down_proj.biases": "model-00196-of-00206.safetensors",
+ "model.layers.35.mlps.0.down_proj.scales": "model-00196-of-00206.safetensors",
+ "model.layers.35.mlps.0.down_proj.weight": "model-00196-of-00206.safetensors",
+ "model.layers.35.mlps.0.gate_proj.biases": "model-00196-of-00206.safetensors",
+ "model.layers.35.mlps.0.gate_proj.scales": "model-00196-of-00206.safetensors",
+ "model.layers.35.mlps.0.gate_proj.weight": "model-00196-of-00206.safetensors",
+ "model.layers.35.mlps.0.up_proj.biases": "model-00196-of-00206.safetensors",
+ "model.layers.35.mlps.0.up_proj.scales": "model-00196-of-00206.safetensors",
+ "model.layers.35.mlps.0.up_proj.weight": "model-00196-of-00206.safetensors",
+ "model.layers.35.mlps.1.down_proj.biases": "model-00196-of-00206.safetensors",
+ "model.layers.35.mlps.1.down_proj.scales": "model-00196-of-00206.safetensors",
+ "model.layers.35.mlps.1.down_proj.weight": "model-00196-of-00206.safetensors",
+ "model.layers.35.mlps.1.gate_proj.biases": "model-00196-of-00206.safetensors",
+ "model.layers.35.mlps.1.gate_proj.scales": "model-00196-of-00206.safetensors",
+ "model.layers.35.mlps.1.gate_proj.weight": "model-00196-of-00206.safetensors",
+ "model.layers.35.mlps.1.up_proj.biases": "model-00196-of-00206.safetensors",
+ "model.layers.35.mlps.1.up_proj.scales": "model-00196-of-00206.safetensors",
+ "model.layers.35.mlps.1.up_proj.weight": "model-00196-of-00206.safetensors",
+ "model.layers.35.post_attention_layernorm.0.weight": "model-00196-of-00206.safetensors",
+ "model.layers.35.post_attention_layernorm.1.weight": "model-00196-of-00206.safetensors",
+ "model.layers.35.self_attn.0.embed_q.biases": "model-00196-of-00206.safetensors",
+ "model.layers.35.self_attn.0.embed_q.scales": "model-00196-of-00206.safetensors",
+ "model.layers.35.self_attn.0.embed_q.weight": "model-00196-of-00206.safetensors",
+ "model.layers.35.self_attn.0.indexer.k_norm.weight": "model-00196-of-00206.safetensors",
+ "model.layers.35.self_attn.0.indexer.weights_proj.biases": "model-00196-of-00206.safetensors",
+ "model.layers.35.self_attn.0.indexer.weights_proj.scales": "model-00196-of-00206.safetensors",
+ "model.layers.35.self_attn.0.indexer.weights_proj.weight": "model-00196-of-00206.safetensors",
+ "model.layers.35.self_attn.0.indexer.wk.biases": "model-00196-of-00206.safetensors",
+ "model.layers.35.self_attn.0.indexer.wk.scales": "model-00196-of-00206.safetensors",
+ "model.layers.35.self_attn.0.indexer.wk.weight": "model-00196-of-00206.safetensors",
+ "model.layers.35.self_attn.0.indexer.wq_b.biases": "model-00196-of-00206.safetensors",
+ "model.layers.35.self_attn.0.indexer.wq_b.scales": "model-00196-of-00206.safetensors",
+ "model.layers.35.self_attn.0.indexer.wq_b.weight": "model-00196-of-00206.safetensors",
+ "model.layers.35.self_attn.0.kv_a_layernorm.weight": "model-00196-of-00206.safetensors",
+ "model.layers.35.self_attn.0.kv_a_proj_with_mqa.biases": "model-00196-of-00206.safetensors",
+ "model.layers.35.self_attn.0.kv_a_proj_with_mqa.scales": "model-00196-of-00206.safetensors",
+ "model.layers.35.self_attn.0.kv_a_proj_with_mqa.weight": "model-00196-of-00206.safetensors",
+ "model.layers.35.self_attn.0.o_proj.biases": "model-00196-of-00206.safetensors",
+ "model.layers.35.self_attn.0.o_proj.scales": "model-00196-of-00206.safetensors",
+ "model.layers.35.self_attn.0.o_proj.weight": "model-00196-of-00206.safetensors",
+ "model.layers.35.self_attn.0.q_a_layernorm.weight": "model-00196-of-00206.safetensors",
+ "model.layers.35.self_attn.0.q_a_proj.biases": "model-00196-of-00206.safetensors",
+ "model.layers.35.self_attn.0.q_a_proj.scales": "model-00196-of-00206.safetensors",
+ "model.layers.35.self_attn.0.q_a_proj.weight": "model-00196-of-00206.safetensors",
+ "model.layers.35.self_attn.0.q_b_proj.biases": "model-00196-of-00206.safetensors",
+ "model.layers.35.self_attn.0.q_b_proj.scales": "model-00196-of-00206.safetensors",
+ "model.layers.35.self_attn.0.q_b_proj.weight": "model-00196-of-00206.safetensors",
+ "model.layers.35.self_attn.0.unembed_out.biases": "model-00196-of-00206.safetensors",
+ "model.layers.35.self_attn.0.unembed_out.scales": "model-00196-of-00206.safetensors",
+ "model.layers.35.self_attn.0.unembed_out.weight": "model-00196-of-00206.safetensors",
+ "model.layers.35.self_attn.1.embed_q.biases": "model-00196-of-00206.safetensors",
+ "model.layers.35.self_attn.1.embed_q.scales": "model-00196-of-00206.safetensors",
+ "model.layers.35.self_attn.1.embed_q.weight": "model-00196-of-00206.safetensors",
+ "model.layers.35.self_attn.1.kv_a_layernorm.weight": "model-00196-of-00206.safetensors",
+ "model.layers.35.self_attn.1.kv_a_proj_with_mqa.biases": "model-00196-of-00206.safetensors",
+ "model.layers.35.self_attn.1.kv_a_proj_with_mqa.scales": "model-00196-of-00206.safetensors",
+ "model.layers.35.self_attn.1.kv_a_proj_with_mqa.weight": "model-00196-of-00206.safetensors",
+ "model.layers.35.self_attn.1.o_proj.biases": "model-00196-of-00206.safetensors",
+ "model.layers.35.self_attn.1.o_proj.scales": "model-00196-of-00206.safetensors",
+ "model.layers.35.self_attn.1.o_proj.weight": "model-00196-of-00206.safetensors",
+ "model.layers.35.self_attn.1.q_a_layernorm.weight": "model-00196-of-00206.safetensors",
+ "model.layers.35.self_attn.1.q_a_proj.biases": "model-00196-of-00206.safetensors",
+ "model.layers.35.self_attn.1.q_a_proj.scales": "model-00196-of-00206.safetensors",
+ "model.layers.35.self_attn.1.q_a_proj.weight": "model-00196-of-00206.safetensors",
+ "model.layers.35.self_attn.1.q_b_proj.biases": "model-00196-of-00206.safetensors",
+ "model.layers.35.self_attn.1.q_b_proj.scales": "model-00196-of-00206.safetensors",
+ "model.layers.35.self_attn.1.q_b_proj.weight": "model-00196-of-00206.safetensors",
+ "model.layers.35.self_attn.1.unembed_out.biases": "model-00196-of-00206.safetensors",
+ "model.layers.35.self_attn.1.unembed_out.scales": "model-00196-of-00206.safetensors",
+ "model.layers.35.self_attn.1.unembed_out.weight": "model-00196-of-00206.safetensors",
+ "model.layers.36.input_layernorm.0.weight": "model-00201-of-00206.safetensors",
+ "model.layers.36.input_layernorm.1.weight": "model-00201-of-00206.safetensors",
+ "model.layers.36.mlp.router.classifier.biases": "model-00201-of-00206.safetensors",
+ "model.layers.36.mlp.router.classifier.scales": "model-00201-of-00206.safetensors",
+ "model.layers.36.mlp.router.classifier.weight": "model-00201-of-00206.safetensors",
+ "model.layers.36.mlp.router.e_score_correction_bias": "model-00201-of-00206.safetensors",
+ "model.layers.36.mlp.switch_mlp.down_proj.biases": "model-00201-of-00206.safetensors",
+ "model.layers.36.mlp.switch_mlp.down_proj.scales": "model-00200-of-00206.safetensors",
+ "model.layers.36.mlp.switch_mlp.down_proj.weight": "model-00200-of-00206.safetensors",
+ "model.layers.36.mlp.switch_mlp.gate_proj.biases": "model-00198-of-00206.safetensors",
+ "model.layers.36.mlp.switch_mlp.gate_proj.scales": "model-00197-of-00206.safetensors",
+ "model.layers.36.mlp.switch_mlp.gate_proj.weight": "model-00197-of-00206.safetensors",
+ "model.layers.36.mlp.switch_mlp.up_proj.biases": "model-00199-of-00206.safetensors",
+ "model.layers.36.mlp.switch_mlp.up_proj.scales": "model-00199-of-00206.safetensors",
+ "model.layers.36.mlp.switch_mlp.up_proj.weight": "model-00198-of-00206.safetensors",
+ "model.layers.36.mlps.0.down_proj.biases": "model-00201-of-00206.safetensors",
+ "model.layers.36.mlps.0.down_proj.scales": "model-00201-of-00206.safetensors",
+ "model.layers.36.mlps.0.down_proj.weight": "model-00201-of-00206.safetensors",
+ "model.layers.36.mlps.0.gate_proj.biases": "model-00201-of-00206.safetensors",
+ "model.layers.36.mlps.0.gate_proj.scales": "model-00201-of-00206.safetensors",
+ "model.layers.36.mlps.0.gate_proj.weight": "model-00201-of-00206.safetensors",
+ "model.layers.36.mlps.0.up_proj.biases": "model-00201-of-00206.safetensors",
+ "model.layers.36.mlps.0.up_proj.scales": "model-00201-of-00206.safetensors",
+ "model.layers.36.mlps.0.up_proj.weight": "model-00201-of-00206.safetensors",
+ "model.layers.36.mlps.1.down_proj.biases": "model-00201-of-00206.safetensors",
+ "model.layers.36.mlps.1.down_proj.scales": "model-00201-of-00206.safetensors",
+ "model.layers.36.mlps.1.down_proj.weight": "model-00201-of-00206.safetensors",
+ "model.layers.36.mlps.1.gate_proj.biases": "model-00201-of-00206.safetensors",
+ "model.layers.36.mlps.1.gate_proj.scales": "model-00201-of-00206.safetensors",
+ "model.layers.36.mlps.1.gate_proj.weight": "model-00201-of-00206.safetensors",
+ "model.layers.36.mlps.1.up_proj.biases": "model-00201-of-00206.safetensors",
+ "model.layers.36.mlps.1.up_proj.scales": "model-00201-of-00206.safetensors",
+ "model.layers.36.mlps.1.up_proj.weight": "model-00201-of-00206.safetensors",
+ "model.layers.36.post_attention_layernorm.0.weight": "model-00201-of-00206.safetensors",
+ "model.layers.36.post_attention_layernorm.1.weight": "model-00201-of-00206.safetensors",
+ "model.layers.36.self_attn.0.embed_q.biases": "model-00201-of-00206.safetensors",
+ "model.layers.36.self_attn.0.embed_q.scales": "model-00201-of-00206.safetensors",
+ "model.layers.36.self_attn.0.embed_q.weight": "model-00201-of-00206.safetensors",
+ "model.layers.36.self_attn.0.indexer.k_norm.weight": "model-00201-of-00206.safetensors",
+ "model.layers.36.self_attn.0.indexer.weights_proj.biases": "model-00201-of-00206.safetensors",
+ "model.layers.36.self_attn.0.indexer.weights_proj.scales": "model-00201-of-00206.safetensors",
+ "model.layers.36.self_attn.0.indexer.weights_proj.weight": "model-00201-of-00206.safetensors",
+ "model.layers.36.self_attn.0.indexer.wk.biases": "model-00201-of-00206.safetensors",
+ "model.layers.36.self_attn.0.indexer.wk.scales": "model-00201-of-00206.safetensors",
+ "model.layers.36.self_attn.0.indexer.wk.weight": "model-00201-of-00206.safetensors",
+ "model.layers.36.self_attn.0.indexer.wq_b.biases": "model-00201-of-00206.safetensors",
+ "model.layers.36.self_attn.0.indexer.wq_b.scales": "model-00201-of-00206.safetensors",
+ "model.layers.36.self_attn.0.indexer.wq_b.weight": "model-00201-of-00206.safetensors",
+ "model.layers.36.self_attn.0.kv_a_layernorm.weight": "model-00201-of-00206.safetensors",
+ "model.layers.36.self_attn.0.kv_a_proj_with_mqa.biases": "model-00201-of-00206.safetensors",
+ "model.layers.36.self_attn.0.kv_a_proj_with_mqa.scales": "model-00201-of-00206.safetensors",
+ "model.layers.36.self_attn.0.kv_a_proj_with_mqa.weight": "model-00201-of-00206.safetensors",
+ "model.layers.36.self_attn.0.o_proj.biases": "model-00201-of-00206.safetensors",
+ "model.layers.36.self_attn.0.o_proj.scales": "model-00201-of-00206.safetensors",
+ "model.layers.36.self_attn.0.o_proj.weight": "model-00201-of-00206.safetensors",
+ "model.layers.36.self_attn.0.q_a_layernorm.weight": "model-00201-of-00206.safetensors",
+ "model.layers.36.self_attn.0.q_a_proj.biases": "model-00201-of-00206.safetensors",
+ "model.layers.36.self_attn.0.q_a_proj.scales": "model-00201-of-00206.safetensors",
+ "model.layers.36.self_attn.0.q_a_proj.weight": "model-00201-of-00206.safetensors",
+ "model.layers.36.self_attn.0.q_b_proj.biases": "model-00201-of-00206.safetensors",
+ "model.layers.36.self_attn.0.q_b_proj.scales": "model-00201-of-00206.safetensors",
+ "model.layers.36.self_attn.0.q_b_proj.weight": "model-00201-of-00206.safetensors",
+ "model.layers.36.self_attn.0.unembed_out.biases": "model-00201-of-00206.safetensors",
+ "model.layers.36.self_attn.0.unembed_out.scales": "model-00201-of-00206.safetensors",
+ "model.layers.36.self_attn.0.unembed_out.weight": "model-00201-of-00206.safetensors",
+ "model.layers.36.self_attn.1.embed_q.biases": "model-00201-of-00206.safetensors",
+ "model.layers.36.self_attn.1.embed_q.scales": "model-00201-of-00206.safetensors",
+ "model.layers.36.self_attn.1.embed_q.weight": "model-00201-of-00206.safetensors",
+ "model.layers.36.self_attn.1.kv_a_layernorm.weight": "model-00201-of-00206.safetensors",
+ "model.layers.36.self_attn.1.kv_a_proj_with_mqa.biases": "model-00201-of-00206.safetensors",
+ "model.layers.36.self_attn.1.kv_a_proj_with_mqa.scales": "model-00201-of-00206.safetensors",
+ "model.layers.36.self_attn.1.kv_a_proj_with_mqa.weight": "model-00201-of-00206.safetensors",
+ "model.layers.36.self_attn.1.o_proj.biases": "model-00201-of-00206.safetensors",
+ "model.layers.36.self_attn.1.o_proj.scales": "model-00201-of-00206.safetensors",
+ "model.layers.36.self_attn.1.o_proj.weight": "model-00201-of-00206.safetensors",
+ "model.layers.36.self_attn.1.q_a_layernorm.weight": "model-00201-of-00206.safetensors",
+ "model.layers.36.self_attn.1.q_a_proj.biases": "model-00201-of-00206.safetensors",
+ "model.layers.36.self_attn.1.q_a_proj.scales": "model-00201-of-00206.safetensors",
+ "model.layers.36.self_attn.1.q_a_proj.weight": "model-00201-of-00206.safetensors",
+ "model.layers.36.self_attn.1.q_b_proj.biases": "model-00201-of-00206.safetensors",
+ "model.layers.36.self_attn.1.q_b_proj.scales": "model-00201-of-00206.safetensors",
+ "model.layers.36.self_attn.1.q_b_proj.weight": "model-00201-of-00206.safetensors",
+ "model.layers.36.self_attn.1.unembed_out.biases": "model-00201-of-00206.safetensors",
+ "model.layers.36.self_attn.1.unembed_out.scales": "model-00201-of-00206.safetensors",
+ "model.layers.36.self_attn.1.unembed_out.weight": "model-00201-of-00206.safetensors",
+ "model.layers.37.input_layernorm.0.weight": "model-00206-of-00206.safetensors",
+ "model.layers.37.input_layernorm.1.weight": "model-00206-of-00206.safetensors",
+ "model.layers.37.mlp.router.classifier.biases": "model-00206-of-00206.safetensors",
+ "model.layers.37.mlp.router.classifier.scales": "model-00206-of-00206.safetensors",
+ "model.layers.37.mlp.router.classifier.weight": "model-00206-of-00206.safetensors",
+ "model.layers.37.mlp.router.e_score_correction_bias": "model-00206-of-00206.safetensors",
+ "model.layers.37.mlp.switch_mlp.down_proj.biases": "model-00206-of-00206.safetensors",
+ "model.layers.37.mlp.switch_mlp.down_proj.scales": "model-00205-of-00206.safetensors",
+ "model.layers.37.mlp.switch_mlp.down_proj.weight": "model-00205-of-00206.safetensors",
+ "model.layers.37.mlp.switch_mlp.gate_proj.biases": "model-00203-of-00206.safetensors",
+ "model.layers.37.mlp.switch_mlp.gate_proj.scales": "model-00202-of-00206.safetensors",
+ "model.layers.37.mlp.switch_mlp.gate_proj.weight": "model-00202-of-00206.safetensors",
+ "model.layers.37.mlp.switch_mlp.up_proj.biases": "model-00204-of-00206.safetensors",
+ "model.layers.37.mlp.switch_mlp.up_proj.scales": "model-00204-of-00206.safetensors",
+ "model.layers.37.mlp.switch_mlp.up_proj.weight": "model-00203-of-00206.safetensors",
+ "model.layers.37.mlps.0.down_proj.biases": "model-00206-of-00206.safetensors",
+ "model.layers.37.mlps.0.down_proj.scales": "model-00206-of-00206.safetensors",
+ "model.layers.37.mlps.0.down_proj.weight": "model-00206-of-00206.safetensors",
+ "model.layers.37.mlps.0.gate_proj.biases": "model-00206-of-00206.safetensors",
+ "model.layers.37.mlps.0.gate_proj.scales": "model-00206-of-00206.safetensors",
+ "model.layers.37.mlps.0.gate_proj.weight": "model-00206-of-00206.safetensors",
+ "model.layers.37.mlps.0.up_proj.biases": "model-00206-of-00206.safetensors",
+ "model.layers.37.mlps.0.up_proj.scales": "model-00206-of-00206.safetensors",
+ "model.layers.37.mlps.0.up_proj.weight": "model-00206-of-00206.safetensors",
+ "model.layers.37.mlps.1.down_proj.biases": "model-00206-of-00206.safetensors",
+ "model.layers.37.mlps.1.down_proj.scales": "model-00206-of-00206.safetensors",
+ "model.layers.37.mlps.1.down_proj.weight": "model-00206-of-00206.safetensors",
+ "model.layers.37.mlps.1.gate_proj.biases": "model-00206-of-00206.safetensors",
+ "model.layers.37.mlps.1.gate_proj.scales": "model-00206-of-00206.safetensors",
+ "model.layers.37.mlps.1.gate_proj.weight": "model-00206-of-00206.safetensors",
+ "model.layers.37.mlps.1.up_proj.biases": "model-00206-of-00206.safetensors",
+ "model.layers.37.mlps.1.up_proj.scales": "model-00206-of-00206.safetensors",
+ "model.layers.37.mlps.1.up_proj.weight": "model-00206-of-00206.safetensors",
+ "model.layers.37.post_attention_layernorm.0.weight": "model-00206-of-00206.safetensors",
+ "model.layers.37.post_attention_layernorm.1.weight": "model-00206-of-00206.safetensors",
+ "model.layers.37.self_attn.0.embed_q.biases": "model-00206-of-00206.safetensors",
+ "model.layers.37.self_attn.0.embed_q.scales": "model-00206-of-00206.safetensors",
+ "model.layers.37.self_attn.0.embed_q.weight": "model-00206-of-00206.safetensors",
+ "model.layers.37.self_attn.0.indexer.k_norm.weight": "model-00206-of-00206.safetensors",
+ "model.layers.37.self_attn.0.indexer.weights_proj.biases": "model-00206-of-00206.safetensors",
+ "model.layers.37.self_attn.0.indexer.weights_proj.scales": "model-00206-of-00206.safetensors",
+ "model.layers.37.self_attn.0.indexer.weights_proj.weight": "model-00206-of-00206.safetensors",
+ "model.layers.37.self_attn.0.indexer.wk.biases": "model-00206-of-00206.safetensors",
+ "model.layers.37.self_attn.0.indexer.wk.scales": "model-00206-of-00206.safetensors",
+ "model.layers.37.self_attn.0.indexer.wk.weight": "model-00206-of-00206.safetensors",
+ "model.layers.37.self_attn.0.indexer.wq_b.biases": "model-00206-of-00206.safetensors",
+ "model.layers.37.self_attn.0.indexer.wq_b.scales": "model-00206-of-00206.safetensors",
+ "model.layers.37.self_attn.0.indexer.wq_b.weight": "model-00206-of-00206.safetensors",
+ "model.layers.37.self_attn.0.kv_a_layernorm.weight": "model-00206-of-00206.safetensors",
+ "model.layers.37.self_attn.0.kv_a_proj_with_mqa.biases": "model-00206-of-00206.safetensors",
+ "model.layers.37.self_attn.0.kv_a_proj_with_mqa.scales": "model-00206-of-00206.safetensors",
+ "model.layers.37.self_attn.0.kv_a_proj_with_mqa.weight": "model-00206-of-00206.safetensors",
+ "model.layers.37.self_attn.0.o_proj.biases": "model-00206-of-00206.safetensors",
+ "model.layers.37.self_attn.0.o_proj.scales": "model-00206-of-00206.safetensors",
+ "model.layers.37.self_attn.0.o_proj.weight": "model-00206-of-00206.safetensors",
+ "model.layers.37.self_attn.0.q_a_layernorm.weight": "model-00206-of-00206.safetensors",
+ "model.layers.37.self_attn.0.q_a_proj.biases": "model-00206-of-00206.safetensors",
+ "model.layers.37.self_attn.0.q_a_proj.scales": "model-00206-of-00206.safetensors",
+ "model.layers.37.self_attn.0.q_a_proj.weight": "model-00206-of-00206.safetensors",
+ "model.layers.37.self_attn.0.q_b_proj.biases": "model-00206-of-00206.safetensors",
+ "model.layers.37.self_attn.0.q_b_proj.scales": "model-00206-of-00206.safetensors",
+ "model.layers.37.self_attn.0.q_b_proj.weight": "model-00206-of-00206.safetensors",
+ "model.layers.37.self_attn.0.unembed_out.biases": "model-00206-of-00206.safetensors",
+ "model.layers.37.self_attn.0.unembed_out.scales": "model-00206-of-00206.safetensors",
+ "model.layers.37.self_attn.0.unembed_out.weight": "model-00206-of-00206.safetensors",
+ "model.layers.37.self_attn.1.embed_q.biases": "model-00206-of-00206.safetensors",
+ "model.layers.37.self_attn.1.embed_q.scales": "model-00206-of-00206.safetensors",
+ "model.layers.37.self_attn.1.embed_q.weight": "model-00206-of-00206.safetensors",
+ "model.layers.37.self_attn.1.kv_a_layernorm.weight": "model-00206-of-00206.safetensors",
+ "model.layers.37.self_attn.1.kv_a_proj_with_mqa.biases": "model-00206-of-00206.safetensors",
+ "model.layers.37.self_attn.1.kv_a_proj_with_mqa.scales": "model-00206-of-00206.safetensors",
+ "model.layers.37.self_attn.1.kv_a_proj_with_mqa.weight": "model-00206-of-00206.safetensors",
+ "model.layers.37.self_attn.1.o_proj.biases": "model-00206-of-00206.safetensors",
+ "model.layers.37.self_attn.1.o_proj.scales": "model-00206-of-00206.safetensors",
+ "model.layers.37.self_attn.1.o_proj.weight": "model-00206-of-00206.safetensors",
+ "model.layers.37.self_attn.1.q_a_layernorm.weight": "model-00206-of-00206.safetensors",
+ "model.layers.37.self_attn.1.q_a_proj.biases": "model-00206-of-00206.safetensors",
+ "model.layers.37.self_attn.1.q_a_proj.scales": "model-00206-of-00206.safetensors",
+ "model.layers.37.self_attn.1.q_a_proj.weight": "model-00206-of-00206.safetensors",
+ "model.layers.37.self_attn.1.q_b_proj.biases": "model-00206-of-00206.safetensors",
+ "model.layers.37.self_attn.1.q_b_proj.scales": "model-00206-of-00206.safetensors",
+ "model.layers.37.self_attn.1.q_b_proj.weight": "model-00206-of-00206.safetensors",
+ "model.layers.37.self_attn.1.unembed_out.biases": "model-00206-of-00206.safetensors",
+ "model.layers.37.self_attn.1.unembed_out.scales": "model-00206-of-00206.safetensors",
+ "model.layers.37.self_attn.1.unembed_out.weight": "model-00206-of-00206.safetensors",
+ "model.layers.4.input_layernorm.0.weight": "model-00041-of-00206.safetensors",
+ "model.layers.4.input_layernorm.1.weight": "model-00041-of-00206.safetensors",
+ "model.layers.4.mlp.router.classifier.biases": "model-00041-of-00206.safetensors",
+ "model.layers.4.mlp.router.classifier.scales": "model-00041-of-00206.safetensors",
+ "model.layers.4.mlp.router.classifier.weight": "model-00041-of-00206.safetensors",
+ "model.layers.4.mlp.router.e_score_correction_bias": "model-00041-of-00206.safetensors",
+ "model.layers.4.mlp.switch_mlp.down_proj.biases": "model-00041-of-00206.safetensors",
+ "model.layers.4.mlp.switch_mlp.down_proj.scales": "model-00040-of-00206.safetensors",
+ "model.layers.4.mlp.switch_mlp.down_proj.weight": "model-00040-of-00206.safetensors",
+ "model.layers.4.mlp.switch_mlp.gate_proj.biases": "model-00038-of-00206.safetensors",
+ "model.layers.4.mlp.switch_mlp.gate_proj.scales": "model-00037-of-00206.safetensors",
+ "model.layers.4.mlp.switch_mlp.gate_proj.weight": "model-00037-of-00206.safetensors",
+ "model.layers.4.mlp.switch_mlp.up_proj.biases": "model-00039-of-00206.safetensors",
+ "model.layers.4.mlp.switch_mlp.up_proj.scales": "model-00039-of-00206.safetensors",
+ "model.layers.4.mlp.switch_mlp.up_proj.weight": "model-00038-of-00206.safetensors",
+ "model.layers.4.mlps.0.down_proj.biases": "model-00041-of-00206.safetensors",
+ "model.layers.4.mlps.0.down_proj.scales": "model-00041-of-00206.safetensors",
+ "model.layers.4.mlps.0.down_proj.weight": "model-00041-of-00206.safetensors",
+ "model.layers.4.mlps.0.gate_proj.biases": "model-00041-of-00206.safetensors",
+ "model.layers.4.mlps.0.gate_proj.scales": "model-00041-of-00206.safetensors",
+ "model.layers.4.mlps.0.gate_proj.weight": "model-00041-of-00206.safetensors",
+ "model.layers.4.mlps.0.up_proj.biases": "model-00041-of-00206.safetensors",
+ "model.layers.4.mlps.0.up_proj.scales": "model-00041-of-00206.safetensors",
+ "model.layers.4.mlps.0.up_proj.weight": "model-00041-of-00206.safetensors",
+ "model.layers.4.mlps.1.down_proj.biases": "model-00041-of-00206.safetensors",
+ "model.layers.4.mlps.1.down_proj.scales": "model-00041-of-00206.safetensors",
+ "model.layers.4.mlps.1.down_proj.weight": "model-00041-of-00206.safetensors",
+ "model.layers.4.mlps.1.gate_proj.biases": "model-00041-of-00206.safetensors",
+ "model.layers.4.mlps.1.gate_proj.scales": "model-00041-of-00206.safetensors",
+ "model.layers.4.mlps.1.gate_proj.weight": "model-00041-of-00206.safetensors",
+ "model.layers.4.mlps.1.up_proj.biases": "model-00041-of-00206.safetensors",
+ "model.layers.4.mlps.1.up_proj.scales": "model-00041-of-00206.safetensors",
+ "model.layers.4.mlps.1.up_proj.weight": "model-00041-of-00206.safetensors",
+ "model.layers.4.post_attention_layernorm.0.weight": "model-00041-of-00206.safetensors",
+ "model.layers.4.post_attention_layernorm.1.weight": "model-00041-of-00206.safetensors",
+ "model.layers.4.self_attn.0.embed_q.biases": "model-00041-of-00206.safetensors",
+ "model.layers.4.self_attn.0.embed_q.scales": "model-00041-of-00206.safetensors",
+ "model.layers.4.self_attn.0.embed_q.weight": "model-00041-of-00206.safetensors",
+ "model.layers.4.self_attn.0.indexer.k_norm.weight": "model-00041-of-00206.safetensors",
+ "model.layers.4.self_attn.0.indexer.weights_proj.biases": "model-00041-of-00206.safetensors",
+ "model.layers.4.self_attn.0.indexer.weights_proj.scales": "model-00041-of-00206.safetensors",
+ "model.layers.4.self_attn.0.indexer.weights_proj.weight": "model-00041-of-00206.safetensors",
+ "model.layers.4.self_attn.0.indexer.wk.biases": "model-00041-of-00206.safetensors",
+ "model.layers.4.self_attn.0.indexer.wk.scales": "model-00041-of-00206.safetensors",
+ "model.layers.4.self_attn.0.indexer.wk.weight": "model-00041-of-00206.safetensors",
+ "model.layers.4.self_attn.0.indexer.wq_b.biases": "model-00041-of-00206.safetensors",
+ "model.layers.4.self_attn.0.indexer.wq_b.scales": "model-00041-of-00206.safetensors",
+ "model.layers.4.self_attn.0.indexer.wq_b.weight": "model-00041-of-00206.safetensors",
+ "model.layers.4.self_attn.0.kv_a_layernorm.weight": "model-00041-of-00206.safetensors",
+ "model.layers.4.self_attn.0.kv_a_proj_with_mqa.biases": "model-00041-of-00206.safetensors",
+ "model.layers.4.self_attn.0.kv_a_proj_with_mqa.scales": "model-00041-of-00206.safetensors",
+ "model.layers.4.self_attn.0.kv_a_proj_with_mqa.weight": "model-00041-of-00206.safetensors",
+ "model.layers.4.self_attn.0.o_proj.biases": "model-00041-of-00206.safetensors",
+ "model.layers.4.self_attn.0.o_proj.scales": "model-00041-of-00206.safetensors",
+ "model.layers.4.self_attn.0.o_proj.weight": "model-00041-of-00206.safetensors",
+ "model.layers.4.self_attn.0.q_a_layernorm.weight": "model-00041-of-00206.safetensors",
+ "model.layers.4.self_attn.0.q_a_proj.biases": "model-00041-of-00206.safetensors",
+ "model.layers.4.self_attn.0.q_a_proj.scales": "model-00041-of-00206.safetensors",
+ "model.layers.4.self_attn.0.q_a_proj.weight": "model-00041-of-00206.safetensors",
+ "model.layers.4.self_attn.0.q_b_proj.biases": "model-00041-of-00206.safetensors",
+ "model.layers.4.self_attn.0.q_b_proj.scales": "model-00041-of-00206.safetensors",
+ "model.layers.4.self_attn.0.q_b_proj.weight": "model-00041-of-00206.safetensors",
+ "model.layers.4.self_attn.0.unembed_out.biases": "model-00041-of-00206.safetensors",
+ "model.layers.4.self_attn.0.unembed_out.scales": "model-00041-of-00206.safetensors",
+ "model.layers.4.self_attn.0.unembed_out.weight": "model-00041-of-00206.safetensors",
+ "model.layers.4.self_attn.1.embed_q.biases": "model-00041-of-00206.safetensors",
+ "model.layers.4.self_attn.1.embed_q.scales": "model-00041-of-00206.safetensors",
+ "model.layers.4.self_attn.1.embed_q.weight": "model-00041-of-00206.safetensors",
+ "model.layers.4.self_attn.1.kv_a_layernorm.weight": "model-00041-of-00206.safetensors",
+ "model.layers.4.self_attn.1.kv_a_proj_with_mqa.biases": "model-00041-of-00206.safetensors",
+ "model.layers.4.self_attn.1.kv_a_proj_with_mqa.scales": "model-00041-of-00206.safetensors",
+ "model.layers.4.self_attn.1.kv_a_proj_with_mqa.weight": "model-00041-of-00206.safetensors",
+ "model.layers.4.self_attn.1.o_proj.biases": "model-00041-of-00206.safetensors",
+ "model.layers.4.self_attn.1.o_proj.scales": "model-00041-of-00206.safetensors",
+ "model.layers.4.self_attn.1.o_proj.weight": "model-00041-of-00206.safetensors",
+ "model.layers.4.self_attn.1.q_a_layernorm.weight": "model-00041-of-00206.safetensors",
+ "model.layers.4.self_attn.1.q_a_proj.biases": "model-00041-of-00206.safetensors",
+ "model.layers.4.self_attn.1.q_a_proj.scales": "model-00041-of-00206.safetensors",
+ "model.layers.4.self_attn.1.q_a_proj.weight": "model-00041-of-00206.safetensors",
+ "model.layers.4.self_attn.1.q_b_proj.biases": "model-00041-of-00206.safetensors",
+ "model.layers.4.self_attn.1.q_b_proj.scales": "model-00041-of-00206.safetensors",
+ "model.layers.4.self_attn.1.q_b_proj.weight": "model-00041-of-00206.safetensors",
+ "model.layers.4.self_attn.1.unembed_out.biases": "model-00041-of-00206.safetensors",
+ "model.layers.4.self_attn.1.unembed_out.scales": "model-00041-of-00206.safetensors",
+ "model.layers.4.self_attn.1.unembed_out.weight": "model-00041-of-00206.safetensors",
+ "model.layers.5.input_layernorm.0.weight": "model-00046-of-00206.safetensors",
+ "model.layers.5.input_layernorm.1.weight": "model-00046-of-00206.safetensors",
+ "model.layers.5.mlp.router.classifier.biases": "model-00046-of-00206.safetensors",
+ "model.layers.5.mlp.router.classifier.scales": "model-00046-of-00206.safetensors",
+ "model.layers.5.mlp.router.classifier.weight": "model-00046-of-00206.safetensors",
+ "model.layers.5.mlp.router.e_score_correction_bias": "model-00046-of-00206.safetensors",
+ "model.layers.5.mlp.switch_mlp.down_proj.biases": "model-00046-of-00206.safetensors",
+ "model.layers.5.mlp.switch_mlp.down_proj.scales": "model-00045-of-00206.safetensors",
+ "model.layers.5.mlp.switch_mlp.down_proj.weight": "model-00045-of-00206.safetensors",
+ "model.layers.5.mlp.switch_mlp.gate_proj.biases": "model-00043-of-00206.safetensors",
+ "model.layers.5.mlp.switch_mlp.gate_proj.scales": "model-00042-of-00206.safetensors",
+ "model.layers.5.mlp.switch_mlp.gate_proj.weight": "model-00042-of-00206.safetensors",
+ "model.layers.5.mlp.switch_mlp.up_proj.biases": "model-00044-of-00206.safetensors",
+ "model.layers.5.mlp.switch_mlp.up_proj.scales": "model-00044-of-00206.safetensors",
+ "model.layers.5.mlp.switch_mlp.up_proj.weight": "model-00043-of-00206.safetensors",
+ "model.layers.5.mlps.0.down_proj.biases": "model-00046-of-00206.safetensors",
+ "model.layers.5.mlps.0.down_proj.scales": "model-00046-of-00206.safetensors",
+ "model.layers.5.mlps.0.down_proj.weight": "model-00046-of-00206.safetensors",
+ "model.layers.5.mlps.0.gate_proj.biases": "model-00046-of-00206.safetensors",
+ "model.layers.5.mlps.0.gate_proj.scales": "model-00046-of-00206.safetensors",
+ "model.layers.5.mlps.0.gate_proj.weight": "model-00046-of-00206.safetensors",
+ "model.layers.5.mlps.0.up_proj.biases": "model-00046-of-00206.safetensors",
+ "model.layers.5.mlps.0.up_proj.scales": "model-00046-of-00206.safetensors",
+ "model.layers.5.mlps.0.up_proj.weight": "model-00046-of-00206.safetensors",
+ "model.layers.5.mlps.1.down_proj.biases": "model-00046-of-00206.safetensors",
+ "model.layers.5.mlps.1.down_proj.scales": "model-00046-of-00206.safetensors",
+ "model.layers.5.mlps.1.down_proj.weight": "model-00046-of-00206.safetensors",
+ "model.layers.5.mlps.1.gate_proj.biases": "model-00046-of-00206.safetensors",
+ "model.layers.5.mlps.1.gate_proj.scales": "model-00046-of-00206.safetensors",
+ "model.layers.5.mlps.1.gate_proj.weight": "model-00046-of-00206.safetensors",
+ "model.layers.5.mlps.1.up_proj.biases": "model-00046-of-00206.safetensors",
+ "model.layers.5.mlps.1.up_proj.scales": "model-00046-of-00206.safetensors",
+ "model.layers.5.mlps.1.up_proj.weight": "model-00046-of-00206.safetensors",
+ "model.layers.5.post_attention_layernorm.0.weight": "model-00046-of-00206.safetensors",
+ "model.layers.5.post_attention_layernorm.1.weight": "model-00046-of-00206.safetensors",
+ "model.layers.5.self_attn.0.embed_q.biases": "model-00046-of-00206.safetensors",
+ "model.layers.5.self_attn.0.embed_q.scales": "model-00046-of-00206.safetensors",
+ "model.layers.5.self_attn.0.embed_q.weight": "model-00046-of-00206.safetensors",
+ "model.layers.5.self_attn.0.indexer.k_norm.weight": "model-00046-of-00206.safetensors",
+ "model.layers.5.self_attn.0.indexer.weights_proj.biases": "model-00046-of-00206.safetensors",
+ "model.layers.5.self_attn.0.indexer.weights_proj.scales": "model-00046-of-00206.safetensors",
+ "model.layers.5.self_attn.0.indexer.weights_proj.weight": "model-00046-of-00206.safetensors",
+ "model.layers.5.self_attn.0.indexer.wk.biases": "model-00046-of-00206.safetensors",
+ "model.layers.5.self_attn.0.indexer.wk.scales": "model-00046-of-00206.safetensors",
+ "model.layers.5.self_attn.0.indexer.wk.weight": "model-00046-of-00206.safetensors",
+ "model.layers.5.self_attn.0.indexer.wq_b.biases": "model-00046-of-00206.safetensors",
+ "model.layers.5.self_attn.0.indexer.wq_b.scales": "model-00046-of-00206.safetensors",
+ "model.layers.5.self_attn.0.indexer.wq_b.weight": "model-00046-of-00206.safetensors",
+ "model.layers.5.self_attn.0.kv_a_layernorm.weight": "model-00046-of-00206.safetensors",
+ "model.layers.5.self_attn.0.kv_a_proj_with_mqa.biases": "model-00046-of-00206.safetensors",
+ "model.layers.5.self_attn.0.kv_a_proj_with_mqa.scales": "model-00046-of-00206.safetensors",
+ "model.layers.5.self_attn.0.kv_a_proj_with_mqa.weight": "model-00046-of-00206.safetensors",
+ "model.layers.5.self_attn.0.o_proj.biases": "model-00046-of-00206.safetensors",
+ "model.layers.5.self_attn.0.o_proj.scales": "model-00046-of-00206.safetensors",
+ "model.layers.5.self_attn.0.o_proj.weight": "model-00046-of-00206.safetensors",
+ "model.layers.5.self_attn.0.q_a_layernorm.weight": "model-00046-of-00206.safetensors",
+ "model.layers.5.self_attn.0.q_a_proj.biases": "model-00046-of-00206.safetensors",
+ "model.layers.5.self_attn.0.q_a_proj.scales": "model-00046-of-00206.safetensors",
+ "model.layers.5.self_attn.0.q_a_proj.weight": "model-00046-of-00206.safetensors",
+ "model.layers.5.self_attn.0.q_b_proj.biases": "model-00046-of-00206.safetensors",
+ "model.layers.5.self_attn.0.q_b_proj.scales": "model-00046-of-00206.safetensors",
+ "model.layers.5.self_attn.0.q_b_proj.weight": "model-00046-of-00206.safetensors",
+ "model.layers.5.self_attn.0.unembed_out.biases": "model-00046-of-00206.safetensors",
+ "model.layers.5.self_attn.0.unembed_out.scales": "model-00046-of-00206.safetensors",
+ "model.layers.5.self_attn.0.unembed_out.weight": "model-00046-of-00206.safetensors",
+ "model.layers.5.self_attn.1.embed_q.biases": "model-00046-of-00206.safetensors",
+ "model.layers.5.self_attn.1.embed_q.scales": "model-00046-of-00206.safetensors",
+ "model.layers.5.self_attn.1.embed_q.weight": "model-00046-of-00206.safetensors",
+ "model.layers.5.self_attn.1.kv_a_layernorm.weight": "model-00046-of-00206.safetensors",
+ "model.layers.5.self_attn.1.kv_a_proj_with_mqa.biases": "model-00046-of-00206.safetensors",
+ "model.layers.5.self_attn.1.kv_a_proj_with_mqa.scales": "model-00046-of-00206.safetensors",
+ "model.layers.5.self_attn.1.kv_a_proj_with_mqa.weight": "model-00046-of-00206.safetensors",
+ "model.layers.5.self_attn.1.o_proj.biases": "model-00046-of-00206.safetensors",
+ "model.layers.5.self_attn.1.o_proj.scales": "model-00046-of-00206.safetensors",
+ "model.layers.5.self_attn.1.o_proj.weight": "model-00046-of-00206.safetensors",
+ "model.layers.5.self_attn.1.q_a_layernorm.weight": "model-00046-of-00206.safetensors",
+ "model.layers.5.self_attn.1.q_a_proj.biases": "model-00046-of-00206.safetensors",
+ "model.layers.5.self_attn.1.q_a_proj.scales": "model-00046-of-00206.safetensors",
+ "model.layers.5.self_attn.1.q_a_proj.weight": "model-00046-of-00206.safetensors",
+ "model.layers.5.self_attn.1.q_b_proj.biases": "model-00046-of-00206.safetensors",
+ "model.layers.5.self_attn.1.q_b_proj.scales": "model-00046-of-00206.safetensors",
+ "model.layers.5.self_attn.1.q_b_proj.weight": "model-00046-of-00206.safetensors",
+ "model.layers.5.self_attn.1.unembed_out.biases": "model-00046-of-00206.safetensors",
+ "model.layers.5.self_attn.1.unembed_out.scales": "model-00046-of-00206.safetensors",
+ "model.layers.5.self_attn.1.unembed_out.weight": "model-00046-of-00206.safetensors",
+ "model.layers.6.input_layernorm.0.weight": "model-00051-of-00206.safetensors",
+ "model.layers.6.input_layernorm.1.weight": "model-00051-of-00206.safetensors",
+ "model.layers.6.mlp.router.classifier.biases": "model-00051-of-00206.safetensors",
+ "model.layers.6.mlp.router.classifier.scales": "model-00051-of-00206.safetensors",
+ "model.layers.6.mlp.router.classifier.weight": "model-00051-of-00206.safetensors",
+ "model.layers.6.mlp.router.e_score_correction_bias": "model-00051-of-00206.safetensors",
+ "model.layers.6.mlp.switch_mlp.down_proj.biases": "model-00051-of-00206.safetensors",
+ "model.layers.6.mlp.switch_mlp.down_proj.scales": "model-00050-of-00206.safetensors",
+ "model.layers.6.mlp.switch_mlp.down_proj.weight": "model-00050-of-00206.safetensors",
+ "model.layers.6.mlp.switch_mlp.gate_proj.biases": "model-00048-of-00206.safetensors",
+ "model.layers.6.mlp.switch_mlp.gate_proj.scales": "model-00047-of-00206.safetensors",
+ "model.layers.6.mlp.switch_mlp.gate_proj.weight": "model-00047-of-00206.safetensors",
+ "model.layers.6.mlp.switch_mlp.up_proj.biases": "model-00049-of-00206.safetensors",
+ "model.layers.6.mlp.switch_mlp.up_proj.scales": "model-00049-of-00206.safetensors",
+ "model.layers.6.mlp.switch_mlp.up_proj.weight": "model-00048-of-00206.safetensors",
+ "model.layers.6.mlps.0.down_proj.biases": "model-00051-of-00206.safetensors",
+ "model.layers.6.mlps.0.down_proj.scales": "model-00051-of-00206.safetensors",
+ "model.layers.6.mlps.0.down_proj.weight": "model-00051-of-00206.safetensors",
+ "model.layers.6.mlps.0.gate_proj.biases": "model-00051-of-00206.safetensors",
+ "model.layers.6.mlps.0.gate_proj.scales": "model-00051-of-00206.safetensors",
+ "model.layers.6.mlps.0.gate_proj.weight": "model-00051-of-00206.safetensors",
+ "model.layers.6.mlps.0.up_proj.biases": "model-00051-of-00206.safetensors",
+ "model.layers.6.mlps.0.up_proj.scales": "model-00051-of-00206.safetensors",
+ "model.layers.6.mlps.0.up_proj.weight": "model-00051-of-00206.safetensors",
+ "model.layers.6.mlps.1.down_proj.biases": "model-00051-of-00206.safetensors",
+ "model.layers.6.mlps.1.down_proj.scales": "model-00051-of-00206.safetensors",
+ "model.layers.6.mlps.1.down_proj.weight": "model-00051-of-00206.safetensors",
+ "model.layers.6.mlps.1.gate_proj.biases": "model-00051-of-00206.safetensors",
+ "model.layers.6.mlps.1.gate_proj.scales": "model-00051-of-00206.safetensors",
+ "model.layers.6.mlps.1.gate_proj.weight": "model-00051-of-00206.safetensors",
+ "model.layers.6.mlps.1.up_proj.biases": "model-00051-of-00206.safetensors",
+ "model.layers.6.mlps.1.up_proj.scales": "model-00051-of-00206.safetensors",
+ "model.layers.6.mlps.1.up_proj.weight": "model-00051-of-00206.safetensors",
+ "model.layers.6.post_attention_layernorm.0.weight": "model-00051-of-00206.safetensors",
+ "model.layers.6.post_attention_layernorm.1.weight": "model-00051-of-00206.safetensors",
+ "model.layers.6.self_attn.0.embed_q.biases": "model-00051-of-00206.safetensors",
+ "model.layers.6.self_attn.0.embed_q.scales": "model-00051-of-00206.safetensors",
+ "model.layers.6.self_attn.0.embed_q.weight": "model-00051-of-00206.safetensors",
+ "model.layers.6.self_attn.0.indexer.k_norm.weight": "model-00051-of-00206.safetensors",
+ "model.layers.6.self_attn.0.indexer.weights_proj.biases": "model-00051-of-00206.safetensors",
+ "model.layers.6.self_attn.0.indexer.weights_proj.scales": "model-00051-of-00206.safetensors",
+ "model.layers.6.self_attn.0.indexer.weights_proj.weight": "model-00051-of-00206.safetensors",
+ "model.layers.6.self_attn.0.indexer.wk.biases": "model-00051-of-00206.safetensors",
+ "model.layers.6.self_attn.0.indexer.wk.scales": "model-00051-of-00206.safetensors",
+ "model.layers.6.self_attn.0.indexer.wk.weight": "model-00051-of-00206.safetensors",
+ "model.layers.6.self_attn.0.indexer.wq_b.biases": "model-00051-of-00206.safetensors",
+ "model.layers.6.self_attn.0.indexer.wq_b.scales": "model-00051-of-00206.safetensors",
+ "model.layers.6.self_attn.0.indexer.wq_b.weight": "model-00051-of-00206.safetensors",
+ "model.layers.6.self_attn.0.kv_a_layernorm.weight": "model-00051-of-00206.safetensors",
+ "model.layers.6.self_attn.0.kv_a_proj_with_mqa.biases": "model-00051-of-00206.safetensors",
+ "model.layers.6.self_attn.0.kv_a_proj_with_mqa.scales": "model-00051-of-00206.safetensors",
+ "model.layers.6.self_attn.0.kv_a_proj_with_mqa.weight": "model-00051-of-00206.safetensors",
+ "model.layers.6.self_attn.0.o_proj.biases": "model-00051-of-00206.safetensors",
+ "model.layers.6.self_attn.0.o_proj.scales": "model-00051-of-00206.safetensors",
+ "model.layers.6.self_attn.0.o_proj.weight": "model-00051-of-00206.safetensors",
+ "model.layers.6.self_attn.0.q_a_layernorm.weight": "model-00051-of-00206.safetensors",
+ "model.layers.6.self_attn.0.q_a_proj.biases": "model-00051-of-00206.safetensors",
+ "model.layers.6.self_attn.0.q_a_proj.scales": "model-00051-of-00206.safetensors",
+ "model.layers.6.self_attn.0.q_a_proj.weight": "model-00051-of-00206.safetensors",
+ "model.layers.6.self_attn.0.q_b_proj.biases": "model-00051-of-00206.safetensors",
+ "model.layers.6.self_attn.0.q_b_proj.scales": "model-00051-of-00206.safetensors",
+ "model.layers.6.self_attn.0.q_b_proj.weight": "model-00051-of-00206.safetensors",
+ "model.layers.6.self_attn.0.unembed_out.biases": "model-00051-of-00206.safetensors",
+ "model.layers.6.self_attn.0.unembed_out.scales": "model-00051-of-00206.safetensors",
+ "model.layers.6.self_attn.0.unembed_out.weight": "model-00051-of-00206.safetensors",
+ "model.layers.6.self_attn.1.embed_q.biases": "model-00051-of-00206.safetensors",
+ "model.layers.6.self_attn.1.embed_q.scales": "model-00051-of-00206.safetensors",
+ "model.layers.6.self_attn.1.embed_q.weight": "model-00051-of-00206.safetensors",
+ "model.layers.6.self_attn.1.kv_a_layernorm.weight": "model-00051-of-00206.safetensors",
+ "model.layers.6.self_attn.1.kv_a_proj_with_mqa.biases": "model-00051-of-00206.safetensors",
+ "model.layers.6.self_attn.1.kv_a_proj_with_mqa.scales": "model-00051-of-00206.safetensors",
+ "model.layers.6.self_attn.1.kv_a_proj_with_mqa.weight": "model-00051-of-00206.safetensors",
+ "model.layers.6.self_attn.1.o_proj.biases": "model-00051-of-00206.safetensors",
+ "model.layers.6.self_attn.1.o_proj.scales": "model-00051-of-00206.safetensors",
+ "model.layers.6.self_attn.1.o_proj.weight": "model-00051-of-00206.safetensors",
+ "model.layers.6.self_attn.1.q_a_layernorm.weight": "model-00051-of-00206.safetensors",
+ "model.layers.6.self_attn.1.q_a_proj.biases": "model-00051-of-00206.safetensors",
+ "model.layers.6.self_attn.1.q_a_proj.scales": "model-00051-of-00206.safetensors",
+ "model.layers.6.self_attn.1.q_a_proj.weight": "model-00051-of-00206.safetensors",
+ "model.layers.6.self_attn.1.q_b_proj.biases": "model-00051-of-00206.safetensors",
+ "model.layers.6.self_attn.1.q_b_proj.scales": "model-00051-of-00206.safetensors",
+ "model.layers.6.self_attn.1.q_b_proj.weight": "model-00051-of-00206.safetensors",
+ "model.layers.6.self_attn.1.unembed_out.biases": "model-00051-of-00206.safetensors",
+ "model.layers.6.self_attn.1.unembed_out.scales": "model-00051-of-00206.safetensors",
+ "model.layers.6.self_attn.1.unembed_out.weight": "model-00051-of-00206.safetensors",
+ "model.layers.7.input_layernorm.0.weight": "model-00056-of-00206.safetensors",
+ "model.layers.7.input_layernorm.1.weight": "model-00056-of-00206.safetensors",
+ "model.layers.7.mlp.router.classifier.biases": "model-00056-of-00206.safetensors",
+ "model.layers.7.mlp.router.classifier.scales": "model-00056-of-00206.safetensors",
+ "model.layers.7.mlp.router.classifier.weight": "model-00056-of-00206.safetensors",
+ "model.layers.7.mlp.router.e_score_correction_bias": "model-00056-of-00206.safetensors",
+ "model.layers.7.mlp.switch_mlp.down_proj.biases": "model-00056-of-00206.safetensors",
+ "model.layers.7.mlp.switch_mlp.down_proj.scales": "model-00055-of-00206.safetensors",
+ "model.layers.7.mlp.switch_mlp.down_proj.weight": "model-00055-of-00206.safetensors",
+ "model.layers.7.mlp.switch_mlp.gate_proj.biases": "model-00053-of-00206.safetensors",
+ "model.layers.7.mlp.switch_mlp.gate_proj.scales": "model-00052-of-00206.safetensors",
+ "model.layers.7.mlp.switch_mlp.gate_proj.weight": "model-00052-of-00206.safetensors",
+ "model.layers.7.mlp.switch_mlp.up_proj.biases": "model-00054-of-00206.safetensors",
+ "model.layers.7.mlp.switch_mlp.up_proj.scales": "model-00054-of-00206.safetensors",
+ "model.layers.7.mlp.switch_mlp.up_proj.weight": "model-00053-of-00206.safetensors",
+ "model.layers.7.mlps.0.down_proj.biases": "model-00056-of-00206.safetensors",
+ "model.layers.7.mlps.0.down_proj.scales": "model-00056-of-00206.safetensors",
+ "model.layers.7.mlps.0.down_proj.weight": "model-00056-of-00206.safetensors",
+ "model.layers.7.mlps.0.gate_proj.biases": "model-00056-of-00206.safetensors",
+ "model.layers.7.mlps.0.gate_proj.scales": "model-00056-of-00206.safetensors",
+ "model.layers.7.mlps.0.gate_proj.weight": "model-00056-of-00206.safetensors",
+ "model.layers.7.mlps.0.up_proj.biases": "model-00056-of-00206.safetensors",
+ "model.layers.7.mlps.0.up_proj.scales": "model-00056-of-00206.safetensors",
+ "model.layers.7.mlps.0.up_proj.weight": "model-00056-of-00206.safetensors",
+ "model.layers.7.mlps.1.down_proj.biases": "model-00056-of-00206.safetensors",
+ "model.layers.7.mlps.1.down_proj.scales": "model-00056-of-00206.safetensors",
+ "model.layers.7.mlps.1.down_proj.weight": "model-00056-of-00206.safetensors",
+ "model.layers.7.mlps.1.gate_proj.biases": "model-00056-of-00206.safetensors",
+ "model.layers.7.mlps.1.gate_proj.scales": "model-00056-of-00206.safetensors",
+ "model.layers.7.mlps.1.gate_proj.weight": "model-00056-of-00206.safetensors",
+ "model.layers.7.mlps.1.up_proj.biases": "model-00056-of-00206.safetensors",
+ "model.layers.7.mlps.1.up_proj.scales": "model-00056-of-00206.safetensors",
+ "model.layers.7.mlps.1.up_proj.weight": "model-00056-of-00206.safetensors",
+ "model.layers.7.post_attention_layernorm.0.weight": "model-00056-of-00206.safetensors",
+ "model.layers.7.post_attention_layernorm.1.weight": "model-00056-of-00206.safetensors",
+ "model.layers.7.self_attn.0.embed_q.biases": "model-00056-of-00206.safetensors",
+ "model.layers.7.self_attn.0.embed_q.scales": "model-00056-of-00206.safetensors",
+ "model.layers.7.self_attn.0.embed_q.weight": "model-00056-of-00206.safetensors",
+ "model.layers.7.self_attn.0.indexer.k_norm.weight": "model-00056-of-00206.safetensors",
+ "model.layers.7.self_attn.0.indexer.weights_proj.biases": "model-00056-of-00206.safetensors",
+ "model.layers.7.self_attn.0.indexer.weights_proj.scales": "model-00056-of-00206.safetensors",
+ "model.layers.7.self_attn.0.indexer.weights_proj.weight": "model-00056-of-00206.safetensors",
+ "model.layers.7.self_attn.0.indexer.wk.biases": "model-00056-of-00206.safetensors",
+ "model.layers.7.self_attn.0.indexer.wk.scales": "model-00056-of-00206.safetensors",
+ "model.layers.7.self_attn.0.indexer.wk.weight": "model-00056-of-00206.safetensors",
+ "model.layers.7.self_attn.0.indexer.wq_b.biases": "model-00056-of-00206.safetensors",
+ "model.layers.7.self_attn.0.indexer.wq_b.scales": "model-00056-of-00206.safetensors",
+ "model.layers.7.self_attn.0.indexer.wq_b.weight": "model-00056-of-00206.safetensors",
+ "model.layers.7.self_attn.0.kv_a_layernorm.weight": "model-00056-of-00206.safetensors",
+ "model.layers.7.self_attn.0.kv_a_proj_with_mqa.biases": "model-00056-of-00206.safetensors",
+ "model.layers.7.self_attn.0.kv_a_proj_with_mqa.scales": "model-00056-of-00206.safetensors",
+ "model.layers.7.self_attn.0.kv_a_proj_with_mqa.weight": "model-00056-of-00206.safetensors",
+ "model.layers.7.self_attn.0.o_proj.biases": "model-00056-of-00206.safetensors",
+ "model.layers.7.self_attn.0.o_proj.scales": "model-00056-of-00206.safetensors",
+ "model.layers.7.self_attn.0.o_proj.weight": "model-00056-of-00206.safetensors",
+ "model.layers.7.self_attn.0.q_a_layernorm.weight": "model-00056-of-00206.safetensors",
+ "model.layers.7.self_attn.0.q_a_proj.biases": "model-00056-of-00206.safetensors",
+ "model.layers.7.self_attn.0.q_a_proj.scales": "model-00056-of-00206.safetensors",
+ "model.layers.7.self_attn.0.q_a_proj.weight": "model-00056-of-00206.safetensors",
+ "model.layers.7.self_attn.0.q_b_proj.biases": "model-00056-of-00206.safetensors",
+ "model.layers.7.self_attn.0.q_b_proj.scales": "model-00056-of-00206.safetensors",
+ "model.layers.7.self_attn.0.q_b_proj.weight": "model-00056-of-00206.safetensors",
+ "model.layers.7.self_attn.0.unembed_out.biases": "model-00056-of-00206.safetensors",
+ "model.layers.7.self_attn.0.unembed_out.scales": "model-00056-of-00206.safetensors",
+ "model.layers.7.self_attn.0.unembed_out.weight": "model-00056-of-00206.safetensors",
+ "model.layers.7.self_attn.1.embed_q.biases": "model-00056-of-00206.safetensors",
+ "model.layers.7.self_attn.1.embed_q.scales": "model-00056-of-00206.safetensors",
+ "model.layers.7.self_attn.1.embed_q.weight": "model-00056-of-00206.safetensors",
+ "model.layers.7.self_attn.1.kv_a_layernorm.weight": "model-00056-of-00206.safetensors",
+ "model.layers.7.self_attn.1.kv_a_proj_with_mqa.biases": "model-00056-of-00206.safetensors",
+ "model.layers.7.self_attn.1.kv_a_proj_with_mqa.scales": "model-00056-of-00206.safetensors",
+ "model.layers.7.self_attn.1.kv_a_proj_with_mqa.weight": "model-00056-of-00206.safetensors",
+ "model.layers.7.self_attn.1.o_proj.biases": "model-00056-of-00206.safetensors",
+ "model.layers.7.self_attn.1.o_proj.scales": "model-00056-of-00206.safetensors",
+ "model.layers.7.self_attn.1.o_proj.weight": "model-00056-of-00206.safetensors",
+ "model.layers.7.self_attn.1.q_a_layernorm.weight": "model-00056-of-00206.safetensors",
+ "model.layers.7.self_attn.1.q_a_proj.biases": "model-00056-of-00206.safetensors",
+ "model.layers.7.self_attn.1.q_a_proj.scales": "model-00056-of-00206.safetensors",
+ "model.layers.7.self_attn.1.q_a_proj.weight": "model-00056-of-00206.safetensors",
+ "model.layers.7.self_attn.1.q_b_proj.biases": "model-00056-of-00206.safetensors",
+ "model.layers.7.self_attn.1.q_b_proj.scales": "model-00056-of-00206.safetensors",
+ "model.layers.7.self_attn.1.q_b_proj.weight": "model-00056-of-00206.safetensors",
+ "model.layers.7.self_attn.1.unembed_out.biases": "model-00056-of-00206.safetensors",
+ "model.layers.7.self_attn.1.unembed_out.scales": "model-00056-of-00206.safetensors",
+ "model.layers.7.self_attn.1.unembed_out.weight": "model-00056-of-00206.safetensors",
+ "model.layers.8.input_layernorm.0.weight": "model-00061-of-00206.safetensors",
+ "model.layers.8.input_layernorm.1.weight": "model-00061-of-00206.safetensors",
+ "model.layers.8.mlp.router.classifier.biases": "model-00061-of-00206.safetensors",
+ "model.layers.8.mlp.router.classifier.scales": "model-00061-of-00206.safetensors",
+ "model.layers.8.mlp.router.classifier.weight": "model-00061-of-00206.safetensors",
+ "model.layers.8.mlp.router.e_score_correction_bias": "model-00061-of-00206.safetensors",
+ "model.layers.8.mlp.switch_mlp.down_proj.biases": "model-00061-of-00206.safetensors",
+ "model.layers.8.mlp.switch_mlp.down_proj.scales": "model-00060-of-00206.safetensors",
+ "model.layers.8.mlp.switch_mlp.down_proj.weight": "model-00060-of-00206.safetensors",
+ "model.layers.8.mlp.switch_mlp.gate_proj.biases": "model-00058-of-00206.safetensors",
+ "model.layers.8.mlp.switch_mlp.gate_proj.scales": "model-00057-of-00206.safetensors",
+ "model.layers.8.mlp.switch_mlp.gate_proj.weight": "model-00057-of-00206.safetensors",
+ "model.layers.8.mlp.switch_mlp.up_proj.biases": "model-00059-of-00206.safetensors",
+ "model.layers.8.mlp.switch_mlp.up_proj.scales": "model-00059-of-00206.safetensors",
+ "model.layers.8.mlp.switch_mlp.up_proj.weight": "model-00058-of-00206.safetensors",
+ "model.layers.8.mlps.0.down_proj.biases": "model-00061-of-00206.safetensors",
+ "model.layers.8.mlps.0.down_proj.scales": "model-00061-of-00206.safetensors",
+ "model.layers.8.mlps.0.down_proj.weight": "model-00061-of-00206.safetensors",
+ "model.layers.8.mlps.0.gate_proj.biases": "model-00061-of-00206.safetensors",
+ "model.layers.8.mlps.0.gate_proj.scales": "model-00061-of-00206.safetensors",
+ "model.layers.8.mlps.0.gate_proj.weight": "model-00061-of-00206.safetensors",
+ "model.layers.8.mlps.0.up_proj.biases": "model-00061-of-00206.safetensors",
+ "model.layers.8.mlps.0.up_proj.scales": "model-00061-of-00206.safetensors",
+ "model.layers.8.mlps.0.up_proj.weight": "model-00061-of-00206.safetensors",
+ "model.layers.8.mlps.1.down_proj.biases": "model-00061-of-00206.safetensors",
+ "model.layers.8.mlps.1.down_proj.scales": "model-00061-of-00206.safetensors",
+ "model.layers.8.mlps.1.down_proj.weight": "model-00061-of-00206.safetensors",
+ "model.layers.8.mlps.1.gate_proj.biases": "model-00061-of-00206.safetensors",
+ "model.layers.8.mlps.1.gate_proj.scales": "model-00061-of-00206.safetensors",
+ "model.layers.8.mlps.1.gate_proj.weight": "model-00061-of-00206.safetensors",
+ "model.layers.8.mlps.1.up_proj.biases": "model-00061-of-00206.safetensors",
+ "model.layers.8.mlps.1.up_proj.scales": "model-00061-of-00206.safetensors",
+ "model.layers.8.mlps.1.up_proj.weight": "model-00061-of-00206.safetensors",
+ "model.layers.8.post_attention_layernorm.0.weight": "model-00061-of-00206.safetensors",
+ "model.layers.8.post_attention_layernorm.1.weight": "model-00061-of-00206.safetensors",
+ "model.layers.8.self_attn.0.embed_q.biases": "model-00061-of-00206.safetensors",
+ "model.layers.8.self_attn.0.embed_q.scales": "model-00061-of-00206.safetensors",
+ "model.layers.8.self_attn.0.embed_q.weight": "model-00061-of-00206.safetensors",
+ "model.layers.8.self_attn.0.indexer.k_norm.weight": "model-00061-of-00206.safetensors",
+ "model.layers.8.self_attn.0.indexer.weights_proj.biases": "model-00061-of-00206.safetensors",
+ "model.layers.8.self_attn.0.indexer.weights_proj.scales": "model-00061-of-00206.safetensors",
+ "model.layers.8.self_attn.0.indexer.weights_proj.weight": "model-00061-of-00206.safetensors",
+ "model.layers.8.self_attn.0.indexer.wk.biases": "model-00061-of-00206.safetensors",
+ "model.layers.8.self_attn.0.indexer.wk.scales": "model-00061-of-00206.safetensors",
+ "model.layers.8.self_attn.0.indexer.wk.weight": "model-00061-of-00206.safetensors",
+ "model.layers.8.self_attn.0.indexer.wq_b.biases": "model-00061-of-00206.safetensors",
+ "model.layers.8.self_attn.0.indexer.wq_b.scales": "model-00061-of-00206.safetensors",
+ "model.layers.8.self_attn.0.indexer.wq_b.weight": "model-00061-of-00206.safetensors",
+ "model.layers.8.self_attn.0.kv_a_layernorm.weight": "model-00061-of-00206.safetensors",
+ "model.layers.8.self_attn.0.kv_a_proj_with_mqa.biases": "model-00061-of-00206.safetensors",
+ "model.layers.8.self_attn.0.kv_a_proj_with_mqa.scales": "model-00061-of-00206.safetensors",
+ "model.layers.8.self_attn.0.kv_a_proj_with_mqa.weight": "model-00061-of-00206.safetensors",
+ "model.layers.8.self_attn.0.o_proj.biases": "model-00061-of-00206.safetensors",
+ "model.layers.8.self_attn.0.o_proj.scales": "model-00061-of-00206.safetensors",
+ "model.layers.8.self_attn.0.o_proj.weight": "model-00061-of-00206.safetensors",
+ "model.layers.8.self_attn.0.q_a_layernorm.weight": "model-00061-of-00206.safetensors",
+ "model.layers.8.self_attn.0.q_a_proj.biases": "model-00061-of-00206.safetensors",
+ "model.layers.8.self_attn.0.q_a_proj.scales": "model-00061-of-00206.safetensors",
+ "model.layers.8.self_attn.0.q_a_proj.weight": "model-00061-of-00206.safetensors",
+ "model.layers.8.self_attn.0.q_b_proj.biases": "model-00061-of-00206.safetensors",
+ "model.layers.8.self_attn.0.q_b_proj.scales": "model-00061-of-00206.safetensors",
+ "model.layers.8.self_attn.0.q_b_proj.weight": "model-00061-of-00206.safetensors",
+ "model.layers.8.self_attn.0.unembed_out.biases": "model-00061-of-00206.safetensors",
+ "model.layers.8.self_attn.0.unembed_out.scales": "model-00061-of-00206.safetensors",
+ "model.layers.8.self_attn.0.unembed_out.weight": "model-00061-of-00206.safetensors",
+ "model.layers.8.self_attn.1.embed_q.biases": "model-00061-of-00206.safetensors",
+ "model.layers.8.self_attn.1.embed_q.scales": "model-00061-of-00206.safetensors",
+ "model.layers.8.self_attn.1.embed_q.weight": "model-00061-of-00206.safetensors",
+ "model.layers.8.self_attn.1.kv_a_layernorm.weight": "model-00061-of-00206.safetensors",
+ "model.layers.8.self_attn.1.kv_a_proj_with_mqa.biases": "model-00061-of-00206.safetensors",
+ "model.layers.8.self_attn.1.kv_a_proj_with_mqa.scales": "model-00061-of-00206.safetensors",
+ "model.layers.8.self_attn.1.kv_a_proj_with_mqa.weight": "model-00061-of-00206.safetensors",
+ "model.layers.8.self_attn.1.o_proj.biases": "model-00061-of-00206.safetensors",
+ "model.layers.8.self_attn.1.o_proj.scales": "model-00061-of-00206.safetensors",
+ "model.layers.8.self_attn.1.o_proj.weight": "model-00061-of-00206.safetensors",
+ "model.layers.8.self_attn.1.q_a_layernorm.weight": "model-00061-of-00206.safetensors",
+ "model.layers.8.self_attn.1.q_a_proj.biases": "model-00061-of-00206.safetensors",
+ "model.layers.8.self_attn.1.q_a_proj.scales": "model-00061-of-00206.safetensors",
+ "model.layers.8.self_attn.1.q_a_proj.weight": "model-00061-of-00206.safetensors",
+ "model.layers.8.self_attn.1.q_b_proj.biases": "model-00061-of-00206.safetensors",
+ "model.layers.8.self_attn.1.q_b_proj.scales": "model-00061-of-00206.safetensors",
+ "model.layers.8.self_attn.1.q_b_proj.weight": "model-00061-of-00206.safetensors",
+ "model.layers.8.self_attn.1.unembed_out.biases": "model-00061-of-00206.safetensors",
+ "model.layers.8.self_attn.1.unembed_out.scales": "model-00061-of-00206.safetensors",
+ "model.layers.8.self_attn.1.unembed_out.weight": "model-00061-of-00206.safetensors",
+ "model.layers.9.input_layernorm.0.weight": "model-00066-of-00206.safetensors",
+ "model.layers.9.input_layernorm.1.weight": "model-00066-of-00206.safetensors",
+ "model.layers.9.mlp.router.classifier.biases": "model-00066-of-00206.safetensors",
+ "model.layers.9.mlp.router.classifier.scales": "model-00066-of-00206.safetensors",
+ "model.layers.9.mlp.router.classifier.weight": "model-00066-of-00206.safetensors",
+ "model.layers.9.mlp.router.e_score_correction_bias": "model-00066-of-00206.safetensors",
+ "model.layers.9.mlp.switch_mlp.down_proj.biases": "model-00066-of-00206.safetensors",
+ "model.layers.9.mlp.switch_mlp.down_proj.scales": "model-00065-of-00206.safetensors",
+ "model.layers.9.mlp.switch_mlp.down_proj.weight": "model-00065-of-00206.safetensors",
+ "model.layers.9.mlp.switch_mlp.gate_proj.biases": "model-00063-of-00206.safetensors",
+ "model.layers.9.mlp.switch_mlp.gate_proj.scales": "model-00062-of-00206.safetensors",
+ "model.layers.9.mlp.switch_mlp.gate_proj.weight": "model-00062-of-00206.safetensors",
+ "model.layers.9.mlp.switch_mlp.up_proj.biases": "model-00064-of-00206.safetensors",
+ "model.layers.9.mlp.switch_mlp.up_proj.scales": "model-00064-of-00206.safetensors",
+ "model.layers.9.mlp.switch_mlp.up_proj.weight": "model-00063-of-00206.safetensors",
+ "model.layers.9.mlps.0.down_proj.biases": "model-00066-of-00206.safetensors",
+ "model.layers.9.mlps.0.down_proj.scales": "model-00066-of-00206.safetensors",
+ "model.layers.9.mlps.0.down_proj.weight": "model-00066-of-00206.safetensors",
+ "model.layers.9.mlps.0.gate_proj.biases": "model-00066-of-00206.safetensors",
+ "model.layers.9.mlps.0.gate_proj.scales": "model-00066-of-00206.safetensors",
+ "model.layers.9.mlps.0.gate_proj.weight": "model-00066-of-00206.safetensors",
+ "model.layers.9.mlps.0.up_proj.biases": "model-00066-of-00206.safetensors",
+ "model.layers.9.mlps.0.up_proj.scales": "model-00066-of-00206.safetensors",
+ "model.layers.9.mlps.0.up_proj.weight": "model-00066-of-00206.safetensors",
+ "model.layers.9.mlps.1.down_proj.biases": "model-00066-of-00206.safetensors",
+ "model.layers.9.mlps.1.down_proj.scales": "model-00066-of-00206.safetensors",
+ "model.layers.9.mlps.1.down_proj.weight": "model-00066-of-00206.safetensors",
+ "model.layers.9.mlps.1.gate_proj.biases": "model-00066-of-00206.safetensors",
+ "model.layers.9.mlps.1.gate_proj.scales": "model-00066-of-00206.safetensors",
+ "model.layers.9.mlps.1.gate_proj.weight": "model-00066-of-00206.safetensors",
+ "model.layers.9.mlps.1.up_proj.biases": "model-00066-of-00206.safetensors",
+ "model.layers.9.mlps.1.up_proj.scales": "model-00066-of-00206.safetensors",
+ "model.layers.9.mlps.1.up_proj.weight": "model-00066-of-00206.safetensors",
+ "model.layers.9.post_attention_layernorm.0.weight": "model-00066-of-00206.safetensors",
+ "model.layers.9.post_attention_layernorm.1.weight": "model-00066-of-00206.safetensors",
+ "model.layers.9.self_attn.0.embed_q.biases": "model-00066-of-00206.safetensors",
+ "model.layers.9.self_attn.0.embed_q.scales": "model-00066-of-00206.safetensors",
+ "model.layers.9.self_attn.0.embed_q.weight": "model-00066-of-00206.safetensors",
+ "model.layers.9.self_attn.0.indexer.k_norm.weight": "model-00066-of-00206.safetensors",
+ "model.layers.9.self_attn.0.indexer.weights_proj.biases": "model-00066-of-00206.safetensors",
+ "model.layers.9.self_attn.0.indexer.weights_proj.scales": "model-00066-of-00206.safetensors",
+ "model.layers.9.self_attn.0.indexer.weights_proj.weight": "model-00066-of-00206.safetensors",
+ "model.layers.9.self_attn.0.indexer.wk.biases": "model-00066-of-00206.safetensors",
+ "model.layers.9.self_attn.0.indexer.wk.scales": "model-00066-of-00206.safetensors",
+ "model.layers.9.self_attn.0.indexer.wk.weight": "model-00066-of-00206.safetensors",
+ "model.layers.9.self_attn.0.indexer.wq_b.biases": "model-00066-of-00206.safetensors",
+ "model.layers.9.self_attn.0.indexer.wq_b.scales": "model-00066-of-00206.safetensors",
+ "model.layers.9.self_attn.0.indexer.wq_b.weight": "model-00066-of-00206.safetensors",
+ "model.layers.9.self_attn.0.kv_a_layernorm.weight": "model-00066-of-00206.safetensors",
+ "model.layers.9.self_attn.0.kv_a_proj_with_mqa.biases": "model-00066-of-00206.safetensors",
+ "model.layers.9.self_attn.0.kv_a_proj_with_mqa.scales": "model-00066-of-00206.safetensors",
+ "model.layers.9.self_attn.0.kv_a_proj_with_mqa.weight": "model-00066-of-00206.safetensors",
+ "model.layers.9.self_attn.0.o_proj.biases": "model-00066-of-00206.safetensors",
+ "model.layers.9.self_attn.0.o_proj.scales": "model-00066-of-00206.safetensors",
+ "model.layers.9.self_attn.0.o_proj.weight": "model-00066-of-00206.safetensors",
+ "model.layers.9.self_attn.0.q_a_layernorm.weight": "model-00066-of-00206.safetensors",
+ "model.layers.9.self_attn.0.q_a_proj.biases": "model-00066-of-00206.safetensors",
+ "model.layers.9.self_attn.0.q_a_proj.scales": "model-00066-of-00206.safetensors",
+ "model.layers.9.self_attn.0.q_a_proj.weight": "model-00066-of-00206.safetensors",
+ "model.layers.9.self_attn.0.q_b_proj.biases": "model-00066-of-00206.safetensors",
+ "model.layers.9.self_attn.0.q_b_proj.scales": "model-00066-of-00206.safetensors",
+ "model.layers.9.self_attn.0.q_b_proj.weight": "model-00066-of-00206.safetensors",
+ "model.layers.9.self_attn.0.unembed_out.biases": "model-00066-of-00206.safetensors",
+ "model.layers.9.self_attn.0.unembed_out.scales": "model-00066-of-00206.safetensors",
+ "model.layers.9.self_attn.0.unembed_out.weight": "model-00066-of-00206.safetensors",
+ "model.layers.9.self_attn.1.embed_q.biases": "model-00066-of-00206.safetensors",
+ "model.layers.9.self_attn.1.embed_q.scales": "model-00066-of-00206.safetensors",
+ "model.layers.9.self_attn.1.embed_q.weight": "model-00066-of-00206.safetensors",
+ "model.layers.9.self_attn.1.kv_a_layernorm.weight": "model-00066-of-00206.safetensors",
+ "model.layers.9.self_attn.1.kv_a_proj_with_mqa.biases": "model-00066-of-00206.safetensors",
+ "model.layers.9.self_attn.1.kv_a_proj_with_mqa.scales": "model-00066-of-00206.safetensors",
+ "model.layers.9.self_attn.1.kv_a_proj_with_mqa.weight": "model-00066-of-00206.safetensors",
+ "model.layers.9.self_attn.1.o_proj.biases": "model-00066-of-00206.safetensors",
+ "model.layers.9.self_attn.1.o_proj.scales": "model-00066-of-00206.safetensors",
+ "model.layers.9.self_attn.1.o_proj.weight": "model-00066-of-00206.safetensors",
+ "model.layers.9.self_attn.1.q_a_layernorm.weight": "model-00066-of-00206.safetensors",
+ "model.layers.9.self_attn.1.q_a_proj.biases": "model-00066-of-00206.safetensors",
+ "model.layers.9.self_attn.1.q_a_proj.scales": "model-00066-of-00206.safetensors",
+ "model.layers.9.self_attn.1.q_a_proj.weight": "model-00066-of-00206.safetensors",
+ "model.layers.9.self_attn.1.q_b_proj.biases": "model-00066-of-00206.safetensors",
+ "model.layers.9.self_attn.1.q_b_proj.scales": "model-00066-of-00206.safetensors",
+ "model.layers.9.self_attn.1.q_b_proj.weight": "model-00066-of-00206.safetensors",
+ "model.layers.9.self_attn.1.unembed_out.biases": "model-00066-of-00206.safetensors",
+ "model.layers.9.self_attn.1.unembed_out.scales": "model-00066-of-00206.safetensors",
+ "model.layers.9.self_attn.1.unembed_out.weight": "model-00066-of-00206.safetensors",
+ "model.ngram_embeddings.embedders.0.biases": "model-00001-of-00206.safetensors",
+ "model.ngram_embeddings.embedders.0.scales": "model-00001-of-00206.safetensors",
+ "model.ngram_embeddings.embedders.0.weight": "model-00001-of-00206.safetensors",
+ "model.ngram_embeddings.embedders.1.biases": "model-00002-of-00206.safetensors",
+ "model.ngram_embeddings.embedders.1.scales": "model-00002-of-00206.safetensors",
+ "model.ngram_embeddings.embedders.1.weight": "model-00002-of-00206.safetensors",
+ "model.ngram_embeddings.embedders.10.biases": "model-00011-of-00206.safetensors",
+ "model.ngram_embeddings.embedders.10.scales": "model-00011-of-00206.safetensors",
+ "model.ngram_embeddings.embedders.10.weight": "model-00011-of-00206.safetensors",
+ "model.ngram_embeddings.embedders.11.biases": "model-00012-of-00206.safetensors",
+ "model.ngram_embeddings.embedders.11.scales": "model-00012-of-00206.safetensors",
+ "model.ngram_embeddings.embedders.11.weight": "model-00012-of-00206.safetensors",
+ "model.ngram_embeddings.embedders.12.biases": "model-00013-of-00206.safetensors",
+ "model.ngram_embeddings.embedders.12.scales": "model-00013-of-00206.safetensors",
+ "model.ngram_embeddings.embedders.12.weight": "model-00013-of-00206.safetensors",
+ "model.ngram_embeddings.embedders.13.biases": "model-00014-of-00206.safetensors",
+ "model.ngram_embeddings.embedders.13.scales": "model-00014-of-00206.safetensors",
+ "model.ngram_embeddings.embedders.13.weight": "model-00014-of-00206.safetensors",
+ "model.ngram_embeddings.embedders.14.biases": "model-00015-of-00206.safetensors",
+ "model.ngram_embeddings.embedders.14.scales": "model-00015-of-00206.safetensors",
+ "model.ngram_embeddings.embedders.14.weight": "model-00015-of-00206.safetensors",
+ "model.ngram_embeddings.embedders.15.biases": "model-00016-of-00206.safetensors",
+ "model.ngram_embeddings.embedders.15.scales": "model-00016-of-00206.safetensors",
+ "model.ngram_embeddings.embedders.15.weight": "model-00016-of-00206.safetensors",
+ "model.ngram_embeddings.embedders.2.biases": "model-00003-of-00206.safetensors",
+ "model.ngram_embeddings.embedders.2.scales": "model-00003-of-00206.safetensors",
+ "model.ngram_embeddings.embedders.2.weight": "model-00003-of-00206.safetensors",
+ "model.ngram_embeddings.embedders.3.biases": "model-00004-of-00206.safetensors",
+ "model.ngram_embeddings.embedders.3.scales": "model-00004-of-00206.safetensors",
+ "model.ngram_embeddings.embedders.3.weight": "model-00004-of-00206.safetensors",
+ "model.ngram_embeddings.embedders.4.biases": "model-00005-of-00206.safetensors",
+ "model.ngram_embeddings.embedders.4.scales": "model-00005-of-00206.safetensors",
+ "model.ngram_embeddings.embedders.4.weight": "model-00005-of-00206.safetensors",
+ "model.ngram_embeddings.embedders.5.biases": "model-00006-of-00206.safetensors",
+ "model.ngram_embeddings.embedders.5.scales": "model-00006-of-00206.safetensors",
+ "model.ngram_embeddings.embedders.5.weight": "model-00006-of-00206.safetensors",
+ "model.ngram_embeddings.embedders.6.biases": "model-00007-of-00206.safetensors",
+ "model.ngram_embeddings.embedders.6.scales": "model-00007-of-00206.safetensors",
+ "model.ngram_embeddings.embedders.6.weight": "model-00007-of-00206.safetensors",
+ "model.ngram_embeddings.embedders.7.biases": "model-00008-of-00206.safetensors",
+ "model.ngram_embeddings.embedders.7.scales": "model-00008-of-00206.safetensors",
+ "model.ngram_embeddings.embedders.7.weight": "model-00008-of-00206.safetensors",
+ "model.ngram_embeddings.embedders.8.biases": "model-00009-of-00206.safetensors",
+ "model.ngram_embeddings.embedders.8.scales": "model-00009-of-00206.safetensors",
+ "model.ngram_embeddings.embedders.8.weight": "model-00009-of-00206.safetensors",
+ "model.ngram_embeddings.embedders.9.biases": "model-00010-of-00206.safetensors",
+ "model.ngram_embeddings.embedders.9.scales": "model-00010-of-00206.safetensors",
+ "model.ngram_embeddings.embedders.9.weight": "model-00010-of-00206.safetensors",
+ "model.ngram_embeddings.post_projs.0.biases": "model-00016-of-00206.safetensors",
+ "model.ngram_embeddings.post_projs.0.scales": "model-00016-of-00206.safetensors",
+ "model.ngram_embeddings.post_projs.0.weight": "model-00016-of-00206.safetensors",
+ "model.ngram_embeddings.post_projs.1.biases": "model-00016-of-00206.safetensors",
+ "model.ngram_embeddings.post_projs.1.scales": "model-00016-of-00206.safetensors",
+ "model.ngram_embeddings.post_projs.1.weight": "model-00016-of-00206.safetensors",
+ "model.ngram_embeddings.post_projs.10.biases": "model-00016-of-00206.safetensors",
+ "model.ngram_embeddings.post_projs.10.scales": "model-00016-of-00206.safetensors",
+ "model.ngram_embeddings.post_projs.10.weight": "model-00016-of-00206.safetensors",
+ "model.ngram_embeddings.post_projs.11.biases": "model-00016-of-00206.safetensors",
+ "model.ngram_embeddings.post_projs.11.scales": "model-00016-of-00206.safetensors",
+ "model.ngram_embeddings.post_projs.11.weight": "model-00016-of-00206.safetensors",
+ "model.ngram_embeddings.post_projs.12.biases": "model-00016-of-00206.safetensors",
+ "model.ngram_embeddings.post_projs.12.scales": "model-00016-of-00206.safetensors",
+ "model.ngram_embeddings.post_projs.12.weight": "model-00016-of-00206.safetensors",
+ "model.ngram_embeddings.post_projs.13.biases": "model-00016-of-00206.safetensors",
+ "model.ngram_embeddings.post_projs.13.scales": "model-00016-of-00206.safetensors",
+ "model.ngram_embeddings.post_projs.13.weight": "model-00016-of-00206.safetensors",
+ "model.ngram_embeddings.post_projs.14.biases": "model-00016-of-00206.safetensors",
+ "model.ngram_embeddings.post_projs.14.scales": "model-00016-of-00206.safetensors",
+ "model.ngram_embeddings.post_projs.14.weight": "model-00016-of-00206.safetensors",
+ "model.ngram_embeddings.post_projs.15.biases": "model-00016-of-00206.safetensors",
+ "model.ngram_embeddings.post_projs.15.scales": "model-00016-of-00206.safetensors",
+ "model.ngram_embeddings.post_projs.15.weight": "model-00016-of-00206.safetensors",
+ "model.ngram_embeddings.post_projs.2.biases": "model-00016-of-00206.safetensors",
+ "model.ngram_embeddings.post_projs.2.scales": "model-00016-of-00206.safetensors",
+ "model.ngram_embeddings.post_projs.2.weight": "model-00016-of-00206.safetensors",
+ "model.ngram_embeddings.post_projs.3.biases": "model-00016-of-00206.safetensors",
+ "model.ngram_embeddings.post_projs.3.scales": "model-00016-of-00206.safetensors",
+ "model.ngram_embeddings.post_projs.3.weight": "model-00016-of-00206.safetensors",
+ "model.ngram_embeddings.post_projs.4.biases": "model-00016-of-00206.safetensors",
+ "model.ngram_embeddings.post_projs.4.scales": "model-00016-of-00206.safetensors",
+ "model.ngram_embeddings.post_projs.4.weight": "model-00016-of-00206.safetensors",
+ "model.ngram_embeddings.post_projs.5.biases": "model-00016-of-00206.safetensors",
+ "model.ngram_embeddings.post_projs.5.scales": "model-00016-of-00206.safetensors",
+ "model.ngram_embeddings.post_projs.5.weight": "model-00016-of-00206.safetensors",
+ "model.ngram_embeddings.post_projs.6.biases": "model-00016-of-00206.safetensors",
+ "model.ngram_embeddings.post_projs.6.scales": "model-00016-of-00206.safetensors",
+ "model.ngram_embeddings.post_projs.6.weight": "model-00016-of-00206.safetensors",
+ "model.ngram_embeddings.post_projs.7.biases": "model-00016-of-00206.safetensors",
+ "model.ngram_embeddings.post_projs.7.scales": "model-00016-of-00206.safetensors",
+ "model.ngram_embeddings.post_projs.7.weight": "model-00016-of-00206.safetensors",
+ "model.ngram_embeddings.post_projs.8.biases": "model-00016-of-00206.safetensors",
+ "model.ngram_embeddings.post_projs.8.scales": "model-00016-of-00206.safetensors",
+ "model.ngram_embeddings.post_projs.8.weight": "model-00016-of-00206.safetensors",
+ "model.ngram_embeddings.post_projs.9.biases": "model-00016-of-00206.safetensors",
+ "model.ngram_embeddings.post_projs.9.scales": "model-00016-of-00206.safetensors",
+ "model.ngram_embeddings.post_projs.9.weight": "model-00016-of-00206.safetensors",
+ "model.ngram_embeddings.word_embeddings.biases": "model-00001-of-00206.safetensors",
+ "model.ngram_embeddings.word_embeddings.scales": "model-00001-of-00206.safetensors",
+ "model.ngram_embeddings.word_embeddings.weight": "model-00001-of-00206.safetensors",
+ "model.norm.weight": "model-00206-of-00206.safetensors"
+ }
+}
\ No newline at end of file
diff --git a/tokenization_llama.py b/tokenization_llama.py
new file mode 100644
index 0000000000000000000000000000000000000000..3fb6e753d15d3fcd096d119e91ac090e86d47bdc
--- /dev/null
+++ b/tokenization_llama.py
@@ -0,0 +1,231 @@
+# coding=utf-8
+# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved.
+#
+# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
+# and OPT implementations in this library. It has been modified from its
+# original forms to accommodate minor architectural differences compared
+# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Tokenization classes for LLaMA."""
+import os
+from shutil import copyfile
+from typing import Any, Dict, List, Optional, Tuple
+
+import sentencepiece as spm
+
+from transformers.tokenization_utils import PreTrainedTokenizer
+from transformers.utils import logging
+
+
+logger = logging.get_logger(__name__)
+
+VOCAB_FILES_NAMES = {"vocab_file": "tokenizer.model"}
+
+PRETRAINED_VOCAB_FILES_MAP = {}
+
+
+class LlamaTokenizer(PreTrainedTokenizer):
+ """
+ Construct a Llama tokenizer. Based on byte-level Byte-Pair-Encoding.
+
+ Args:
+ vocab_file (`str`):
+ Path to the vocabulary file.
+ """
+
+ vocab_files_names = VOCAB_FILES_NAMES
+ pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
+ model_input_names = ["input_ids", "attention_mask"]
+
+ def __init__(
+ self,
+ vocab_file,
+ unk_token="",
+ bos_token="",
+ eos_token="",
+ sp_model_kwargs: Optional[Dict[str, Any]] = None,
+ add_bos_token=True,
+ add_eos_token=False,
+ decode_with_prefix_space=False,
+ **kwargs,
+ ):
+ self.sp_model_kwargs = {} if sp_model_kwargs is None else sp_model_kwargs
+ super().__init__(bos_token=bos_token, eos_token=eos_token, unk_token=unk_token, **kwargs)
+ self.vocab_file = vocab_file
+ self.add_bos_token = add_bos_token
+ self.add_eos_token = add_eos_token
+ self.decode_with_prefix_space = decode_with_prefix_space
+ self.sp_model = spm.SentencePieceProcessor(**self.sp_model_kwargs)
+ self.sp_model.Load(vocab_file)
+ self._no_prefix_space_tokens = None
+
+ """ Initialisation"""
+
+ @property
+ def no_prefix_space_tokens(self):
+ if self._no_prefix_space_tokens is None:
+ vocab = self.convert_ids_to_tokens(list(range(self.vocab_size)))
+ self._no_prefix_space_tokens = {i for i, tok in enumerate(vocab) if not tok.startswith("▁")}
+ return self._no_prefix_space_tokens
+
+ @property
+ def vocab_size(self):
+ """Returns vocab size"""
+ return self.sp_model.get_piece_size()
+
+ @property
+ def bos_token_id(self) -> Optional[int]:
+ return self.sp_model.bos_id()
+
+ @property
+ def eos_token_id(self) -> Optional[int]:
+ return self.sp_model.eos_id()
+
+ def get_vocab(self):
+ """Returns vocab as a dict"""
+ vocab = {self.convert_ids_to_tokens(i): i for i in range(self.vocab_size)}
+ vocab.update(self.added_tokens_encoder)
+ return vocab
+
+ def _tokenize(self, text):
+ """Returns a tokenized string."""
+ return self.sp_model.encode(text, out_type=str)
+
+ def _convert_token_to_id(self, token):
+ """Converts a token (str) in an id using the vocab."""
+ return self.sp_model.piece_to_id(token)
+
+ def _convert_id_to_token(self, index):
+ """Converts an index (integer) in a token (str) using the vocab."""
+ token = self.sp_model.IdToPiece(index)
+ return token
+
+ def _maybe_add_prefix_space(self, tokens, decoded):
+ if tokens and tokens[0] not in self.no_prefix_space_tokens:
+ return " " + decoded
+ else:
+ return decoded
+
+ def convert_tokens_to_string(self, tokens):
+ """Converts a sequence of tokens (string) in a single string."""
+ current_sub_tokens = []
+ out_string = ""
+ prev_is_special = False
+ for token in tokens:
+ # make sure that special tokens are not decoded using sentencepiece model
+ if token in self.all_special_tokens:
+ if not prev_is_special:
+ out_string += " "
+ out_string += self.sp_model.decode(current_sub_tokens) + token
+ prev_is_special = True
+ current_sub_tokens = []
+ else:
+ current_sub_tokens.append(token)
+ prev_is_special = False
+ out_string += self.sp_model.decode(current_sub_tokens)
+ out_string = self._maybe_add_prefix_space(tokens=tokens, decoded=out_string)
+ return out_string
+
+ def save_vocabulary(self, save_directory, filename_prefix: Optional[str] = None) -> Tuple[str]:
+ """
+ Save the vocabulary and special tokens file to a directory.
+
+ Args:
+ save_directory (`str`):
+ The directory in which to save the vocabulary.
+
+ Returns:
+ `Tuple(str)`: Paths to the files saved.
+ """
+ if not os.path.isdir(save_directory):
+ logger.error(f"Vocabulary path ({save_directory}) should be a directory")
+ return
+ out_vocab_file = os.path.join(
+ save_directory, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]
+ )
+
+ if os.path.abspath(self.vocab_file) != os.path.abspath(out_vocab_file) and os.path.isfile(self.vocab_file):
+ copyfile(self.vocab_file, out_vocab_file)
+ elif not os.path.isfile(self.vocab_file):
+ with open(out_vocab_file, "wb") as fi:
+ content_spiece_model = self.sp_model.serialized_model_proto()
+ fi.write(content_spiece_model)
+
+ return (out_vocab_file,)
+
+ def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None):
+ if self.add_bos_token:
+ bos_token_ids = [self.bos_token_id]
+ else:
+ bos_token_ids = []
+
+ output = bos_token_ids + token_ids_0
+
+ if token_ids_1 is not None:
+ output = output + token_ids_1
+
+ if self.add_eos_token:
+ output = output + [self.eos_token_id]
+
+ return output
+
+ def get_special_tokens_mask(
+ self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None, already_has_special_tokens: bool = False
+ ) -> List[int]:
+ """
+ Retrieve sequence ids from a token list that has no special tokens added. This method is called when adding
+ special tokens using the tokenizer `prepare_for_model` method.
+
+ Args:
+ token_ids_0 (`List[int]`):
+ List of IDs.
+ token_ids_1 (`List[int]`, *optional*):
+ Optional second list of IDs for sequence pairs.
+ already_has_special_tokens (`bool`, *optional*, defaults to `False`):
+ Whether or not the token list is already formatted with special tokens for the model.
+
+ Returns:
+ `List[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.
+ """
+ if already_has_special_tokens:
+ return super().get_special_tokens_mask(
+ token_ids_0=token_ids_0, token_ids_1=token_ids_1, already_has_special_tokens=True
+ )
+
+ if token_ids_1 is None:
+ return [1] + ([0] * len(token_ids_0)) + [1]
+ return [1] + ([0] * len(token_ids_0)) + [1, 1] + ([0] * len(token_ids_1)) + [1]
+
+ def create_token_type_ids_from_sequences(
+ self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None
+ ) -> List[int]:
+ """
+ Create a mask from the two sequences passed to be used in a sequence-pair classification task. T5 does not make
+ use of token type ids, therefore a list of zeros is returned.
+
+ Args:
+ token_ids_0 (`List[int]`):
+ List of IDs.
+ token_ids_1 (`List[int]`, *optional*):
+ Optional second list of IDs for sequence pairs.
+
+ Returns:
+ `List[int]`: List of zeros.
+ """
+ eos = [self.eos_token_id]
+
+ if token_ids_1 is None:
+ return len(token_ids_0 + eos) * [0]
+ return len(token_ids_0 + eos + token_ids_1 + eos) * [0]
diff --git a/tokenizer.json b/tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..167254a07853107fe4d9fe8b1693dd1d627d921d
--- /dev/null
+++ b/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c0859e010a8f5a1429944ed76d93dfd0b06e3862e97ab6fda743dd5184fe6240
+size 12741264
diff --git a/tokenizer_config.json b/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..dbe86a6961a4855a588d4b705d21579f38806ee4
--- /dev/null
+++ b/tokenizer_config.json
@@ -0,0 +1,15 @@
+{
+ "add_prefix_space": false,
+ "backend": "tokenizers",
+ "bos_token": "",
+ "clean_up_tokenization_spaces": false,
+ "eos_token": "",
+ "is_local": true,
+ "local_files_only": false,
+ "model_max_length": 131072,
+ "pad_token": "",
+ "sp_model_kwargs": {},
+ "tokenizer_class": "TokenizersBackend",
+ "tool_parser_type": "longcat",
+ "unk_token": ""
+}