diff --git "a/axquant_plan.json" "b/axquant_plan.json" --- "a/axquant_plan.json" +++ "b/axquant_plan.json" @@ -1,8 +1,8 @@ { - "analysis_sha256": "adcabd6cfe28d52f7d575002c4dd9598af1bf7ff2302d2573a6461aa4a70f623", + "analysis_sha256": "970a8f05f20eb0bb2df3ab73cdd8dd5fb31df2de312257c1beb2e335f9c8660b", "architecture_profile": { "adapter_id": "qwen36-v1", - "config_model_type": "qwen3_5", + "config_model_type": "qwen3_5_moe", "dense": false, "mtp_declared": true, "notes": [ @@ -1005,31 +1005,31 @@ "tensor": "model.language_model.layers.0.mlp.shared_expert.up_proj.weight" }, { - "bits": 6, - "group_size": 64, + "bits": 8, + "group_size": 32, "method": "affine", "metrics": { - "cosine_distance": 0.04275, + "cosine_distance": 0.021876116042958818, "decode_latency_cost": 0.0, - "hidden_state_error": 0.07600000000000001, - "long_context_loss": 0.028499999999999998, - "mtp_acceptance_loss": 0.0076, - "output_kl": 0.095, + "hidden_state_error": 0.03889087296526012, + "long_context_loss": 0.014584077361972545, + "mtp_acceptance_loss": 0.003889087296526012, + "output_kl": 0.04861359120657515, "peak_memory_cost": 0.0, "prefill_latency_cost": 0.0, - "task_loss_delta": 0.061750000000000006, - "token_disagreement": 0.052250000000000005 + "task_loss_delta": 0.03159883428427385, + "token_disagreement": 0.026737475163616336 }, "module_path": "model.language_model.layers.0.mlp.shared_expert_gate", "outlier_strategy": "none", "parameters": 2048, - "predicted_loss": 0.05187, - "reason": "selected by marginal quality gain per storage bit", - "role": "expert", + "predicted_loss": 0.02654302079879003, + "reason": "protected router policy", + "role": "router", "scale_strategy": "group-affine", "strategy_metadata": { - "selected_from_candidates": 7, - "storage_bpw": 6.5 + "selected_from_candidates": 3, + "storage_bpw": 9.0 }, "tensor": "model.language_model.layers.0.mlp.shared_expert_gate.weight" }, @@ -1382,31 +1382,31 @@ "tensor": "model.language_model.layers.1.mlp.shared_expert.up_proj.weight" }, { - "bits": 6, - "group_size": 64, + "bits": 8, + "group_size": 32, "method": "affine", "metrics": { - "cosine_distance": 0.04275, + "cosine_distance": 0.021876116042958818, "decode_latency_cost": 0.0, - "hidden_state_error": 0.07600000000000001, - "long_context_loss": 0.028499999999999998, - "mtp_acceptance_loss": 0.0076, - "output_kl": 0.095, + "hidden_state_error": 0.03889087296526012, + "long_context_loss": 0.014584077361972545, + "mtp_acceptance_loss": 0.003889087296526012, + "output_kl": 0.04861359120657515, "peak_memory_cost": 0.0, "prefill_latency_cost": 0.0, - "task_loss_delta": 0.061750000000000006, - "token_disagreement": 0.052250000000000005 + "task_loss_delta": 0.03159883428427385, + "token_disagreement": 0.026737475163616336 }, "module_path": "model.language_model.layers.1.mlp.shared_expert_gate", "outlier_strategy": "none", "parameters": 2048, - "predicted_loss": 0.05187, - "reason": "selected by marginal quality gain per storage bit", - "role": "expert", + "predicted_loss": 0.02654302079879003, + "reason": "protected router policy", + "role": "router", "scale_strategy": "group-affine", "strategy_metadata": { - "selected_from_candidates": 7, - "storage_bpw": 6.5 + "selected_from_candidates": 3, + "storage_bpw": 9.0 }, "tensor": "model.language_model.layers.1.mlp.shared_expert_gate.weight" }, @@ -11213,31 +11213,31 @@ "tensor": "model.language_model.layers.2.mlp.shared_expert.up_proj.weight" }, { - "bits": 6, - "group_size": 64, + "bits": 8, + "group_size": 32, "method": "affine", "metrics": { - "cosine_distance": 0.04275, + "cosine_distance": 0.021876116042958818, "decode_latency_cost": 0.0, - "hidden_state_error": 0.07600000000000001, - "long_context_loss": 0.028499999999999998, - "mtp_acceptance_loss": 0.0076, - "output_kl": 0.095, + "hidden_state_error": 0.03889087296526012, + "long_context_loss": 0.014584077361972545, + "mtp_acceptance_loss": 0.003889087296526012, + "output_kl": 0.04861359120657515, "peak_memory_cost": 0.0, "prefill_latency_cost": 0.0, - "task_loss_delta": 0.061750000000000006, - "token_disagreement": 0.052250000000000005 + "task_loss_delta": 0.03159883428427385, + "token_disagreement": 0.026737475163616336 }, "module_path": "model.language_model.layers.2.mlp.shared_expert_gate", "outlier_strategy": "none", "parameters": 2048, - "predicted_loss": 0.05187, - "reason": "selected by marginal quality gain per storage bit", - "role": "expert", + "predicted_loss": 0.02654302079879003, + "reason": "protected router policy", + "role": "router", "scale_strategy": "group-affine", "strategy_metadata": { - "selected_from_candidates": 7, - "storage_bpw": 6.5 + "selected_from_candidates": 3, + "storage_bpw": 9.0 }, "tensor": "model.language_model.layers.2.mlp.shared_expert_gate.weight" }, @@ -11474,31 +11474,31 @@ "tensor": "model.language_model.layers.3.mlp.shared_expert.up_proj.weight" }, { - "bits": 6, + "bits": 8, "group_size": 32, "method": "affine", "metrics": { - "cosine_distance": 0.03022881489572491, + "cosine_distance": 0.021876116042958818, "decode_latency_cost": 0.0, - "hidden_state_error": 0.05374011537017762, - "long_context_loss": 0.020152543263816602, - "mtp_acceptance_loss": 0.005374011537017762, - "output_kl": 0.06717514421272203, + "hidden_state_error": 0.03889087296526012, + "long_context_loss": 0.014584077361972545, + "mtp_acceptance_loss": 0.003889087296526012, + "output_kl": 0.04861359120657515, "peak_memory_cost": 0.0, "prefill_latency_cost": 0.0, - "task_loss_delta": 0.043663843738269315, - "token_disagreement": 0.036946329316997116 + "task_loss_delta": 0.03159883428427385, + "token_disagreement": 0.026737475163616336 }, "module_path": "model.language_model.layers.3.mlp.shared_expert_gate", "outlier_strategy": "none", "parameters": 2048, - "predicted_loss": 0.036677628740146226, - "reason": "selected by marginal quality gain per storage bit", - "role": "expert", + "predicted_loss": 0.02654302079879003, + "reason": "protected router policy", + "role": "router", "scale_strategy": "group-affine", "strategy_metadata": { - "selected_from_candidates": 7, - "storage_bpw": 7.0 + "selected_from_candidates": 3, + "storage_bpw": 9.0 }, "tensor": "model.language_model.layers.3.mlp.shared_expert_gate.weight" }, @@ -12170,31 +12170,31 @@ "tensor": "model.language_model.layers.4.mlp.shared_expert.up_proj.weight" }, { - "bits": 6, + "bits": 8, "group_size": 32, "method": "affine", "metrics": { - "cosine_distance": 0.03022881489572491, + "cosine_distance": 0.021876116042958818, "decode_latency_cost": 0.0, - "hidden_state_error": 0.05374011537017762, - "long_context_loss": 0.020152543263816602, - "mtp_acceptance_loss": 0.005374011537017762, - "output_kl": 0.06717514421272203, + "hidden_state_error": 0.03889087296526012, + "long_context_loss": 0.014584077361972545, + "mtp_acceptance_loss": 0.003889087296526012, + "output_kl": 0.04861359120657515, "peak_memory_cost": 0.0, "prefill_latency_cost": 0.0, - "task_loss_delta": 0.043663843738269315, - "token_disagreement": 0.036946329316997116 + "task_loss_delta": 0.03159883428427385, + "token_disagreement": 0.026737475163616336 }, "module_path": "model.language_model.layers.4.mlp.shared_expert_gate", "outlier_strategy": "none", "parameters": 2048, - "predicted_loss": 0.036677628740146226, - "reason": "selected by marginal quality gain per storage bit", - "role": "expert", + "predicted_loss": 0.02654302079879003, + "reason": "protected router policy", + "role": "router", "scale_strategy": "group-affine", "strategy_metadata": { - "selected_from_candidates": 7, - "storage_bpw": 7.0 + "selected_from_candidates": 3, + "storage_bpw": 9.0 }, "tensor": "model.language_model.layers.4.mlp.shared_expert_gate.weight" }, @@ -12692,31 +12692,31 @@ "tensor": "model.language_model.layers.5.mlp.shared_expert.up_proj.weight" }, { - "bits": 6, + "bits": 8, "group_size": 32, "method": "affine", "metrics": { - "cosine_distance": 0.03022881489572491, + "cosine_distance": 0.021876116042958818, "decode_latency_cost": 0.0, - "hidden_state_error": 0.05374011537017762, - "long_context_loss": 0.020152543263816602, - "mtp_acceptance_loss": 0.005374011537017762, - "output_kl": 0.06717514421272203, + "hidden_state_error": 0.03889087296526012, + "long_context_loss": 0.014584077361972545, + "mtp_acceptance_loss": 0.003889087296526012, + "output_kl": 0.04861359120657515, "peak_memory_cost": 0.0, "prefill_latency_cost": 0.0, - "task_loss_delta": 0.043663843738269315, - "token_disagreement": 0.036946329316997116 + "task_loss_delta": 0.03159883428427385, + "token_disagreement": 0.026737475163616336 }, "module_path": "model.language_model.layers.5.mlp.shared_expert_gate", "outlier_strategy": "none", "parameters": 2048, - "predicted_loss": 0.036677628740146226, - "reason": "selected by marginal quality gain per storage bit", - "role": "expert", + "predicted_loss": 0.02654302079879003, + "reason": "protected router policy", + "role": "router", "scale_strategy": "group-affine", "strategy_metadata": { - "selected_from_candidates": 7, - "storage_bpw": 7.0 + "selected_from_candidates": 3, + "storage_bpw": 9.0 }, "tensor": "model.language_model.layers.5.mlp.shared_expert_gate.weight" }, @@ -13214,31 +13214,31 @@ "tensor": "model.language_model.layers.6.mlp.shared_expert.up_proj.weight" }, { - "bits": 6, + "bits": 8, "group_size": 32, "method": "affine", "metrics": { - "cosine_distance": 0.03022881489572491, + "cosine_distance": 0.021876116042958818, "decode_latency_cost": 0.0, - "hidden_state_error": 0.05374011537017762, - "long_context_loss": 0.020152543263816602, - "mtp_acceptance_loss": 0.005374011537017762, - "output_kl": 0.06717514421272203, + "hidden_state_error": 0.03889087296526012, + "long_context_loss": 0.014584077361972545, + "mtp_acceptance_loss": 0.003889087296526012, + "output_kl": 0.04861359120657515, "peak_memory_cost": 0.0, "prefill_latency_cost": 0.0, - "task_loss_delta": 0.043663843738269315, - "token_disagreement": 0.036946329316997116 + "task_loss_delta": 0.03159883428427385, + "token_disagreement": 0.026737475163616336 }, "module_path": "model.language_model.layers.6.mlp.shared_expert_gate", "outlier_strategy": "none", "parameters": 2048, - "predicted_loss": 0.036677628740146226, - "reason": "selected by marginal quality gain per storage bit", - "role": "expert", + "predicted_loss": 0.02654302079879003, + "reason": "protected router policy", + "role": "router", "scale_strategy": "group-affine", "strategy_metadata": { - "selected_from_candidates": 7, - "storage_bpw": 7.0 + "selected_from_candidates": 3, + "storage_bpw": 9.0 }, "tensor": "model.language_model.layers.6.mlp.shared_expert_gate.weight" }, @@ -13475,31 +13475,31 @@ "tensor": "model.language_model.layers.7.mlp.shared_expert.up_proj.weight" }, { - "bits": 6, + "bits": 8, "group_size": 32, "method": "affine", "metrics": { - "cosine_distance": 0.03022881489572491, + "cosine_distance": 0.021876116042958818, "decode_latency_cost": 0.0, - "hidden_state_error": 0.05374011537017762, - "long_context_loss": 0.020152543263816602, - "mtp_acceptance_loss": 0.005374011537017762, - "output_kl": 0.06717514421272203, + "hidden_state_error": 0.03889087296526012, + "long_context_loss": 0.014584077361972545, + "mtp_acceptance_loss": 0.003889087296526012, + "output_kl": 0.04861359120657515, "peak_memory_cost": 0.0, "prefill_latency_cost": 0.0, - "task_loss_delta": 0.043663843738269315, - "token_disagreement": 0.036946329316997116 + "task_loss_delta": 0.03159883428427385, + "token_disagreement": 0.026737475163616336 }, "module_path": "model.language_model.layers.7.mlp.shared_expert_gate", "outlier_strategy": "none", "parameters": 2048, - "predicted_loss": 0.036677628740146226, - "reason": "selected by marginal quality gain per storage bit", - "role": "expert", + "predicted_loss": 0.02654302079879003, + "reason": "protected router policy", + "role": "router", "scale_strategy": "group-affine", "strategy_metadata": { - "selected_from_candidates": 7, - "storage_bpw": 7.0 + "selected_from_candidates": 3, + "storage_bpw": 9.0 }, "tensor": "model.language_model.layers.7.mlp.shared_expert_gate.weight" }, @@ -14461,31 +14461,31 @@ "tensor": "model.language_model.layers.10.mlp.shared_expert.up_proj.weight" }, { - "bits": 6, + "bits": 8, "group_size": 32, "method": "affine", "metrics": { - "cosine_distance": 0.03022881489572491, + "cosine_distance": 0.021876116042958818, "decode_latency_cost": 0.0, - "hidden_state_error": 0.05374011537017762, - "long_context_loss": 0.020152543263816602, - "mtp_acceptance_loss": 0.005374011537017762, - "output_kl": 0.06717514421272203, + "hidden_state_error": 0.03889087296526012, + "long_context_loss": 0.014584077361972545, + "mtp_acceptance_loss": 0.003889087296526012, + "output_kl": 0.04861359120657515, "peak_memory_cost": 0.0, "prefill_latency_cost": 0.0, - "task_loss_delta": 0.043663843738269315, - "token_disagreement": 0.036946329316997116 + "task_loss_delta": 0.03159883428427385, + "token_disagreement": 0.026737475163616336 }, "module_path": "model.language_model.layers.10.mlp.shared_expert_gate", "outlier_strategy": "none", "parameters": 2048, - "predicted_loss": 0.036677628740146226, - "reason": "selected by marginal quality gain per storage bit", - "role": "expert", + "predicted_loss": 0.02654302079879003, + "reason": "protected router policy", + "role": "router", "scale_strategy": "group-affine", "strategy_metadata": { - "selected_from_candidates": 7, - "storage_bpw": 7.0 + "selected_from_candidates": 3, + "storage_bpw": 9.0 }, "tensor": "model.language_model.layers.10.mlp.shared_expert_gate.weight" }, @@ -14838,31 +14838,31 @@ "tensor": "model.language_model.layers.8.mlp.shared_expert.up_proj.weight" }, { - "bits": 6, + "bits": 8, "group_size": 32, "method": "affine", "metrics": { - "cosine_distance": 0.03022881489572491, + "cosine_distance": 0.021876116042958818, "decode_latency_cost": 0.0, - "hidden_state_error": 0.05374011537017762, - "long_context_loss": 0.020152543263816602, - "mtp_acceptance_loss": 0.005374011537017762, - "output_kl": 0.06717514421272203, + "hidden_state_error": 0.03889087296526012, + "long_context_loss": 0.014584077361972545, + "mtp_acceptance_loss": 0.003889087296526012, + "output_kl": 0.04861359120657515, "peak_memory_cost": 0.0, "prefill_latency_cost": 0.0, - "task_loss_delta": 0.043663843738269315, - "token_disagreement": 0.036946329316997116 + "task_loss_delta": 0.03159883428427385, + "token_disagreement": 0.026737475163616336 }, "module_path": "model.language_model.layers.8.mlp.shared_expert_gate", "outlier_strategy": "none", "parameters": 2048, - "predicted_loss": 0.036677628740146226, - "reason": "selected by marginal quality gain per storage bit", - "role": "expert", + "predicted_loss": 0.02654302079879003, + "reason": "protected router policy", + "role": "router", "scale_strategy": "group-affine", "strategy_metadata": { - "selected_from_candidates": 7, - "storage_bpw": 7.0 + "selected_from_candidates": 3, + "storage_bpw": 9.0 }, "tensor": "model.language_model.layers.8.mlp.shared_expert_gate.weight" }, @@ -15215,31 +15215,31 @@ "tensor": "model.language_model.layers.9.mlp.shared_expert.up_proj.weight" }, { - "bits": 6, + "bits": 8, "group_size": 32, "method": "affine", "metrics": { - "cosine_distance": 0.03022881489572491, + "cosine_distance": 0.021876116042958818, "decode_latency_cost": 0.0, - "hidden_state_error": 0.05374011537017762, - "long_context_loss": 0.020152543263816602, - "mtp_acceptance_loss": 0.005374011537017762, - "output_kl": 0.06717514421272203, + "hidden_state_error": 0.03889087296526012, + "long_context_loss": 0.014584077361972545, + "mtp_acceptance_loss": 0.003889087296526012, + "output_kl": 0.04861359120657515, "peak_memory_cost": 0.0, "prefill_latency_cost": 0.0, - "task_loss_delta": 0.043663843738269315, - "token_disagreement": 0.036946329316997116 + "task_loss_delta": 0.03159883428427385, + "token_disagreement": 0.026737475163616336 }, "module_path": "model.language_model.layers.9.mlp.shared_expert_gate", "outlier_strategy": "none", "parameters": 2048, - "predicted_loss": 0.036677628740146226, - "reason": "selected by marginal quality gain per storage bit", - "role": "expert", + "predicted_loss": 0.02654302079879003, + "reason": "protected router policy", + "role": "router", "scale_strategy": "group-affine", "strategy_metadata": { - "selected_from_candidates": 7, - "storage_bpw": 7.0 + "selected_from_candidates": 3, + "storage_bpw": 9.0 }, "tensor": "model.language_model.layers.9.mlp.shared_expert_gate.weight" }, @@ -15853,31 +15853,31 @@ "tensor": "model.language_model.layers.11.mlp.shared_expert.up_proj.weight" }, { - "bits": 6, + "bits": 8, "group_size": 32, "method": "affine", "metrics": { - "cosine_distance": 0.03022881489572491, + "cosine_distance": 0.021876116042958818, "decode_latency_cost": 0.0, - "hidden_state_error": 0.05374011537017762, - "long_context_loss": 0.020152543263816602, - "mtp_acceptance_loss": 0.005374011537017762, - "output_kl": 0.06717514421272203, + "hidden_state_error": 0.03889087296526012, + "long_context_loss": 0.014584077361972545, + "mtp_acceptance_loss": 0.003889087296526012, + "output_kl": 0.04861359120657515, "peak_memory_cost": 0.0, "prefill_latency_cost": 0.0, - "task_loss_delta": 0.043663843738269315, - "token_disagreement": 0.036946329316997116 + "task_loss_delta": 0.03159883428427385, + "token_disagreement": 0.026737475163616336 }, "module_path": "model.language_model.layers.11.mlp.shared_expert_gate", "outlier_strategy": "none", "parameters": 2048, - "predicted_loss": 0.036677628740146226, - "reason": "selected by marginal quality gain per storage bit", - "role": "expert", + "predicted_loss": 0.02654302079879003, + "reason": "protected router policy", + "role": "router", "scale_strategy": "group-affine", "strategy_metadata": { - "selected_from_candidates": 7, - "storage_bpw": 7.0 + "selected_from_candidates": 3, + "storage_bpw": 9.0 }, "tensor": "model.language_model.layers.11.mlp.shared_expert_gate.weight" }, @@ -16317,31 +16317,31 @@ "tensor": "model.language_model.layers.12.mlp.shared_expert.up_proj.weight" }, { - "bits": 6, + "bits": 8, "group_size": 32, "method": "affine", "metrics": { - "cosine_distance": 0.03022881489572491, + "cosine_distance": 0.021876116042958818, "decode_latency_cost": 0.0, - "hidden_state_error": 0.05374011537017762, - "long_context_loss": 0.020152543263816602, - "mtp_acceptance_loss": 0.005374011537017762, - "output_kl": 0.06717514421272203, + "hidden_state_error": 0.03889087296526012, + "long_context_loss": 0.014584077361972545, + "mtp_acceptance_loss": 0.003889087296526012, + "output_kl": 0.04861359120657515, "peak_memory_cost": 0.0, "prefill_latency_cost": 0.0, - "task_loss_delta": 0.043663843738269315, - "token_disagreement": 0.036946329316997116 + "task_loss_delta": 0.03159883428427385, + "token_disagreement": 0.026737475163616336 }, "module_path": "model.language_model.layers.12.mlp.shared_expert_gate", "outlier_strategy": "none", "parameters": 2048, - "predicted_loss": 0.036677628740146226, - "reason": "selected by marginal quality gain per storage bit", - "role": "expert", + "predicted_loss": 0.02654302079879003, + "reason": "protected router policy", + "role": "router", "scale_strategy": "group-affine", "strategy_metadata": { - "selected_from_candidates": 7, - "storage_bpw": 7.0 + "selected_from_candidates": 3, + "storage_bpw": 9.0 }, "tensor": "model.language_model.layers.12.mlp.shared_expert_gate.weight" }, @@ -16694,31 +16694,31 @@ "tensor": "model.language_model.layers.13.mlp.shared_expert.up_proj.weight" }, { - "bits": 6, + "bits": 8, "group_size": 32, "method": "affine", "metrics": { - "cosine_distance": 0.03022881489572491, + "cosine_distance": 0.021876116042958818, "decode_latency_cost": 0.0, - "hidden_state_error": 0.05374011537017762, - "long_context_loss": 0.020152543263816602, - "mtp_acceptance_loss": 0.005374011537017762, - "output_kl": 0.06717514421272203, + "hidden_state_error": 0.03889087296526012, + "long_context_loss": 0.014584077361972545, + "mtp_acceptance_loss": 0.003889087296526012, + "output_kl": 0.04861359120657515, "peak_memory_cost": 0.0, "prefill_latency_cost": 0.0, - "task_loss_delta": 0.043663843738269315, - "token_disagreement": 0.036946329316997116 + "task_loss_delta": 0.03159883428427385, + "token_disagreement": 0.026737475163616336 }, "module_path": "model.language_model.layers.13.mlp.shared_expert_gate", "outlier_strategy": "none", "parameters": 2048, - "predicted_loss": 0.036677628740146226, - "reason": "selected by marginal quality gain per storage bit", - "role": "expert", + "predicted_loss": 0.02654302079879003, + "reason": "protected router policy", + "role": "router", "scale_strategy": "group-affine", "strategy_metadata": { - "selected_from_candidates": 7, - "storage_bpw": 7.0 + "selected_from_candidates": 3, + "storage_bpw": 9.0 }, "tensor": "model.language_model.layers.13.mlp.shared_expert_gate.weight" }, @@ -17303,31 +17303,31 @@ "tensor": "model.language_model.layers.14.linear_attn.in_proj_a.weight" }, { - "bits": 6, - "group_size": 32, + "bits": 8, + "group_size": 64, "method": "affine", "metrics": { - "cosine_distance": 0.057275649276110355, + "cosine_distance": 0.02025, "decode_latency_cost": 0.0, - "hidden_state_error": 0.10182337649086284, - "long_context_loss": 0.15909902576697318, - "mtp_acceptance_loss": 0.010182337649086284, - "output_kl": 0.12727922061357855, + "hidden_state_error": 0.036, + "long_context_loss": 0.056249999999999994, + "mtp_acceptance_loss": 0.0036, + "output_kl": 0.045, "peak_memory_cost": 0.0, "prefill_latency_cost": 0.0, - "task_loss_delta": 0.08273149339882606, - "token_disagreement": 0.0700035713374682 + "task_loss_delta": 0.029249999999999998, + "token_disagreement": 0.02475 }, "module_path": "model.language_model.layers.14.linear_attn.in_proj_b", "outlier_strategy": "none", "parameters": 65536, - "predicted_loss": 0.07916767522164586, + "predicted_loss": 0.027989999999999998, "reason": "selected by marginal quality gain per storage bit", "role": "attention", "scale_strategy": "group-affine", "strategy_metadata": { "selected_from_candidates": 7, - "storage_bpw": 7.0 + "storage_bpw": 8.5 }, "tensor": "model.language_model.layers.14.linear_attn.in_proj_b.weight" }, @@ -17477,31 +17477,31 @@ "tensor": "model.language_model.layers.14.mlp.shared_expert.up_proj.weight" }, { - "bits": 6, + "bits": 8, "group_size": 32, "method": "affine", "metrics": { - "cosine_distance": 0.03022881489572491, + "cosine_distance": 0.021876116042958818, "decode_latency_cost": 0.0, - "hidden_state_error": 0.05374011537017762, - "long_context_loss": 0.020152543263816602, - "mtp_acceptance_loss": 0.005374011537017762, - "output_kl": 0.06717514421272203, + "hidden_state_error": 0.03889087296526012, + "long_context_loss": 0.014584077361972545, + "mtp_acceptance_loss": 0.003889087296526012, + "output_kl": 0.04861359120657515, "peak_memory_cost": 0.0, "prefill_latency_cost": 0.0, - "task_loss_delta": 0.043663843738269315, - "token_disagreement": 0.036946329316997116 + "task_loss_delta": 0.03159883428427385, + "token_disagreement": 0.026737475163616336 }, "module_path": "model.language_model.layers.14.mlp.shared_expert_gate", "outlier_strategy": "none", "parameters": 2048, - "predicted_loss": 0.036677628740146226, - "reason": "selected by marginal quality gain per storage bit", - "role": "expert", + "predicted_loss": 0.02654302079879003, + "reason": "protected router policy", + "role": "router", "scale_strategy": "group-affine", "strategy_metadata": { - "selected_from_candidates": 7, - "storage_bpw": 7.0 + "selected_from_candidates": 3, + "storage_bpw": 9.0 }, "tensor": "model.language_model.layers.14.mlp.shared_expert_gate.weight" }, @@ -17709,31 +17709,31 @@ "tensor": "model.language_model.layers.15.mlp.shared_expert.up_proj.weight" }, { - "bits": 6, + "bits": 8, "group_size": 32, "method": "affine", "metrics": { - "cosine_distance": 0.03022881489572491, + "cosine_distance": 0.021876116042958818, "decode_latency_cost": 0.0, - "hidden_state_error": 0.05374011537017762, - "long_context_loss": 0.020152543263816602, - "mtp_acceptance_loss": 0.005374011537017762, - "output_kl": 0.06717514421272203, + "hidden_state_error": 0.03889087296526012, + "long_context_loss": 0.014584077361972545, + "mtp_acceptance_loss": 0.003889087296526012, + "output_kl": 0.04861359120657515, "peak_memory_cost": 0.0, "prefill_latency_cost": 0.0, - "task_loss_delta": 0.043663843738269315, - "token_disagreement": 0.036946329316997116 + "task_loss_delta": 0.03159883428427385, + "token_disagreement": 0.026737475163616336 }, "module_path": "model.language_model.layers.15.mlp.shared_expert_gate", "outlier_strategy": "none", "parameters": 2048, - "predicted_loss": 0.036677628740146226, - "reason": "selected by marginal quality gain per storage bit", - "role": "expert", + "predicted_loss": 0.02654302079879003, + "reason": "protected router policy", + "role": "router", "scale_strategy": "group-affine", "strategy_metadata": { - "selected_from_candidates": 7, - "storage_bpw": 7.0 + "selected_from_candidates": 3, + "storage_bpw": 9.0 }, "tensor": "model.language_model.layers.15.mlp.shared_expert_gate.weight" }, @@ -17941,60 +17941,60 @@ "tensor": "model.language_model.layers.16.linear_attn.dt_bias" }, { - "bits": 6, - "group_size": 32, + "bits": 8, + "group_size": 64, "method": "affine", "metrics": { - "cosine_distance": 0.057275649276110355, + "cosine_distance": 0.02025, "decode_latency_cost": 0.0, - "hidden_state_error": 0.10182337649086284, - "long_context_loss": 0.15909902576697318, - "mtp_acceptance_loss": 0.010182337649086284, - "output_kl": 0.12727922061357855, + "hidden_state_error": 0.036, + "long_context_loss": 0.056249999999999994, + "mtp_acceptance_loss": 0.0036, + "output_kl": 0.045, "peak_memory_cost": 0.0, "prefill_latency_cost": 0.0, - "task_loss_delta": 0.08273149339882606, - "token_disagreement": 0.0700035713374682 + "task_loss_delta": 0.029249999999999998, + "token_disagreement": 0.02475 }, "module_path": "model.language_model.layers.16.linear_attn.in_proj_a", "outlier_strategy": "none", "parameters": 65536, - "predicted_loss": 0.07916767522164586, + "predicted_loss": 0.027989999999999998, "reason": "selected by marginal quality gain per storage bit", "role": "attention", "scale_strategy": "group-affine", "strategy_metadata": { "selected_from_candidates": 7, - "storage_bpw": 7.0 + "storage_bpw": 8.5 }, "tensor": "model.language_model.layers.16.linear_attn.in_proj_a.weight" }, { - "bits": 6, - "group_size": 32, + "bits": 8, + "group_size": 64, "method": "affine", "metrics": { - "cosine_distance": 0.057275649276110355, + "cosine_distance": 0.02025, "decode_latency_cost": 0.0, - "hidden_state_error": 0.10182337649086284, - "long_context_loss": 0.15909902576697318, - "mtp_acceptance_loss": 0.010182337649086284, - "output_kl": 0.12727922061357855, + "hidden_state_error": 0.036, + "long_context_loss": 0.056249999999999994, + "mtp_acceptance_loss": 0.0036, + "output_kl": 0.045, "peak_memory_cost": 0.0, "prefill_latency_cost": 0.0, - "task_loss_delta": 0.08273149339882606, - "token_disagreement": 0.0700035713374682 + "task_loss_delta": 0.029249999999999998, + "token_disagreement": 0.02475 }, "module_path": "model.language_model.layers.16.linear_attn.in_proj_b", "outlier_strategy": "none", "parameters": 65536, - "predicted_loss": 0.07916767522164586, + "predicted_loss": 0.027989999999999998, "reason": "selected by marginal quality gain per storage bit", "role": "attention", "scale_strategy": "group-affine", "strategy_metadata": { "selected_from_candidates": 7, - "storage_bpw": 7.0 + "storage_bpw": 8.5 }, "tensor": "model.language_model.layers.16.linear_attn.in_proj_b.weight" }, @@ -18173,31 +18173,31 @@ "tensor": "model.language_model.layers.16.mlp.shared_expert.up_proj.weight" }, { - "bits": 6, + "bits": 8, "group_size": 32, "method": "affine", "metrics": { - "cosine_distance": 0.03022881489572491, + "cosine_distance": 0.021876116042958818, "decode_latency_cost": 0.0, - "hidden_state_error": 0.05374011537017762, - "long_context_loss": 0.020152543263816602, - "mtp_acceptance_loss": 0.005374011537017762, - "output_kl": 0.06717514421272203, + "hidden_state_error": 0.03889087296526012, + "long_context_loss": 0.014584077361972545, + "mtp_acceptance_loss": 0.003889087296526012, + "output_kl": 0.04861359120657515, "peak_memory_cost": 0.0, "prefill_latency_cost": 0.0, - "task_loss_delta": 0.043663843738269315, - "token_disagreement": 0.036946329316997116 + "task_loss_delta": 0.03159883428427385, + "token_disagreement": 0.026737475163616336 }, "module_path": "model.language_model.layers.16.mlp.shared_expert_gate", "outlier_strategy": "none", "parameters": 2048, - "predicted_loss": 0.036677628740146226, - "reason": "selected by marginal quality gain per storage bit", - "role": "expert", + "predicted_loss": 0.02654302079879003, + "reason": "protected router policy", + "role": "router", "scale_strategy": "group-affine", "strategy_metadata": { - "selected_from_candidates": 7, - "storage_bpw": 7.0 + "selected_from_candidates": 3, + "storage_bpw": 9.0 }, "tensor": "model.language_model.layers.16.mlp.shared_expert_gate.weight" }, @@ -18753,60 +18753,60 @@ "tensor": "model.language_model.layers.17.linear_attn.dt_bias" }, { - "bits": 6, - "group_size": 32, + "bits": 8, + "group_size": 64, "method": "affine", "metrics": { - "cosine_distance": 0.057275649276110355, + "cosine_distance": 0.02025, "decode_latency_cost": 0.0, - "hidden_state_error": 0.10182337649086284, - "long_context_loss": 0.15909902576697318, - "mtp_acceptance_loss": 0.010182337649086284, - "output_kl": 0.12727922061357855, + "hidden_state_error": 0.036, + "long_context_loss": 0.056249999999999994, + "mtp_acceptance_loss": 0.0036, + "output_kl": 0.045, "peak_memory_cost": 0.0, "prefill_latency_cost": 0.0, - "task_loss_delta": 0.08273149339882606, - "token_disagreement": 0.0700035713374682 + "task_loss_delta": 0.029249999999999998, + "token_disagreement": 0.02475 }, "module_path": "model.language_model.layers.17.linear_attn.in_proj_a", "outlier_strategy": "none", "parameters": 65536, - "predicted_loss": 0.07916767522164586, + "predicted_loss": 0.027989999999999998, "reason": "selected by marginal quality gain per storage bit", "role": "attention", "scale_strategy": "group-affine", "strategy_metadata": { "selected_from_candidates": 7, - "storage_bpw": 7.0 + "storage_bpw": 8.5 }, "tensor": "model.language_model.layers.17.linear_attn.in_proj_a.weight" }, { - "bits": 6, - "group_size": 32, + "bits": 8, + "group_size": 64, "method": "affine", "metrics": { - "cosine_distance": 0.057275649276110355, + "cosine_distance": 0.02025, "decode_latency_cost": 0.0, - "hidden_state_error": 0.10182337649086284, - "long_context_loss": 0.15909902576697318, - "mtp_acceptance_loss": 0.010182337649086284, - "output_kl": 0.12727922061357855, + "hidden_state_error": 0.036, + "long_context_loss": 0.056249999999999994, + "mtp_acceptance_loss": 0.0036, + "output_kl": 0.045, "peak_memory_cost": 0.0, "prefill_latency_cost": 0.0, - "task_loss_delta": 0.08273149339882606, - "token_disagreement": 0.0700035713374682 + "task_loss_delta": 0.029249999999999998, + "token_disagreement": 0.02475 }, "module_path": "model.language_model.layers.17.linear_attn.in_proj_b", "outlier_strategy": "none", "parameters": 65536, - "predicted_loss": 0.07916767522164586, + "predicted_loss": 0.027989999999999998, "reason": "selected by marginal quality gain per storage bit", "role": "attention", "scale_strategy": "group-affine", "strategy_metadata": { "selected_from_candidates": 7, - "storage_bpw": 7.0 + "storage_bpw": 8.5 }, "tensor": "model.language_model.layers.17.linear_attn.in_proj_b.weight" }, @@ -18956,31 +18956,31 @@ "tensor": "model.language_model.layers.17.mlp.shared_expert.up_proj.weight" }, { - "bits": 6, + "bits": 8, "group_size": 32, "method": "affine", "metrics": { - "cosine_distance": 0.03022881489572491, + "cosine_distance": 0.021876116042958818, "decode_latency_cost": 0.0, - "hidden_state_error": 0.05374011537017762, - "long_context_loss": 0.020152543263816602, - "mtp_acceptance_loss": 0.005374011537017762, - "output_kl": 0.06717514421272203, + "hidden_state_error": 0.03889087296526012, + "long_context_loss": 0.014584077361972545, + "mtp_acceptance_loss": 0.003889087296526012, + "output_kl": 0.04861359120657515, "peak_memory_cost": 0.0, "prefill_latency_cost": 0.0, - "task_loss_delta": 0.043663843738269315, - "token_disagreement": 0.036946329316997116 + "task_loss_delta": 0.03159883428427385, + "token_disagreement": 0.026737475163616336 }, "module_path": "model.language_model.layers.17.mlp.shared_expert_gate", "outlier_strategy": "none", "parameters": 2048, - "predicted_loss": 0.036677628740146226, - "reason": "selected by marginal quality gain per storage bit", - "role": "expert", + "predicted_loss": 0.02654302079879003, + "reason": "protected router policy", + "role": "router", "scale_strategy": "group-affine", "strategy_metadata": { - "selected_from_candidates": 7, - "storage_bpw": 7.0 + "selected_from_candidates": 3, + "storage_bpw": 9.0 }, "tensor": "model.language_model.layers.17.mlp.shared_expert_gate.weight" }, @@ -19130,60 +19130,60 @@ "tensor": "model.language_model.layers.18.linear_attn.dt_bias" }, { - "bits": 6, - "group_size": 32, + "bits": 8, + "group_size": 64, "method": "affine", "metrics": { - "cosine_distance": 0.057275649276110355, + "cosine_distance": 0.02025, "decode_latency_cost": 0.0, - "hidden_state_error": 0.10182337649086284, - "long_context_loss": 0.15909902576697318, - "mtp_acceptance_loss": 0.010182337649086284, - "output_kl": 0.12727922061357855, + "hidden_state_error": 0.036, + "long_context_loss": 0.056249999999999994, + "mtp_acceptance_loss": 0.0036, + "output_kl": 0.045, "peak_memory_cost": 0.0, "prefill_latency_cost": 0.0, - "task_loss_delta": 0.08273149339882606, - "token_disagreement": 0.0700035713374682 + "task_loss_delta": 0.029249999999999998, + "token_disagreement": 0.02475 }, "module_path": "model.language_model.layers.18.linear_attn.in_proj_a", "outlier_strategy": "none", "parameters": 65536, - "predicted_loss": 0.07916767522164586, + "predicted_loss": 0.027989999999999998, "reason": "selected by marginal quality gain per storage bit", "role": "attention", "scale_strategy": "group-affine", "strategy_metadata": { "selected_from_candidates": 7, - "storage_bpw": 7.0 + "storage_bpw": 8.5 }, "tensor": "model.language_model.layers.18.linear_attn.in_proj_a.weight" }, { - "bits": 6, - "group_size": 32, + "bits": 8, + "group_size": 64, "method": "affine", "metrics": { - "cosine_distance": 0.057275649276110355, + "cosine_distance": 0.02025, "decode_latency_cost": 0.0, - "hidden_state_error": 0.10182337649086284, - "long_context_loss": 0.15909902576697318, - "mtp_acceptance_loss": 0.010182337649086284, - "output_kl": 0.12727922061357855, + "hidden_state_error": 0.036, + "long_context_loss": 0.056249999999999994, + "mtp_acceptance_loss": 0.0036, + "output_kl": 0.045, "peak_memory_cost": 0.0, "prefill_latency_cost": 0.0, - "task_loss_delta": 0.08273149339882606, - "token_disagreement": 0.0700035713374682 + "task_loss_delta": 0.029249999999999998, + "token_disagreement": 0.02475 }, "module_path": "model.language_model.layers.18.linear_attn.in_proj_b", "outlier_strategy": "none", "parameters": 65536, - "predicted_loss": 0.07916767522164586, + "predicted_loss": 0.027989999999999998, "reason": "selected by marginal quality gain per storage bit", "role": "attention", "scale_strategy": "group-affine", "strategy_metadata": { "selected_from_candidates": 7, - "storage_bpw": 7.0 + "storage_bpw": 8.5 }, "tensor": "model.language_model.layers.18.linear_attn.in_proj_b.weight" }, @@ -19362,31 +19362,31 @@ "tensor": "model.language_model.layers.18.mlp.shared_expert.up_proj.weight" }, { - "bits": 6, + "bits": 8, "group_size": 32, "method": "affine", "metrics": { - "cosine_distance": 0.03022881489572491, + "cosine_distance": 0.021876116042958818, "decode_latency_cost": 0.0, - "hidden_state_error": 0.05374011537017762, - "long_context_loss": 0.020152543263816602, - "mtp_acceptance_loss": 0.005374011537017762, - "output_kl": 0.06717514421272203, + "hidden_state_error": 0.03889087296526012, + "long_context_loss": 0.014584077361972545, + "mtp_acceptance_loss": 0.003889087296526012, + "output_kl": 0.04861359120657515, "peak_memory_cost": 0.0, "prefill_latency_cost": 0.0, - "task_loss_delta": 0.043663843738269315, - "token_disagreement": 0.036946329316997116 + "task_loss_delta": 0.03159883428427385, + "token_disagreement": 0.026737475163616336 }, "module_path": "model.language_model.layers.18.mlp.shared_expert_gate", "outlier_strategy": "none", "parameters": 2048, - "predicted_loss": 0.036677628740146226, - "reason": "selected by marginal quality gain per storage bit", - "role": "expert", + "predicted_loss": 0.02654302079879003, + "reason": "protected router policy", + "role": "router", "scale_strategy": "group-affine", "strategy_metadata": { - "selected_from_candidates": 7, - "storage_bpw": 7.0 + "selected_from_candidates": 3, + "storage_bpw": 9.0 }, "tensor": "model.language_model.layers.18.mlp.shared_expert_gate.weight" }, @@ -19594,31 +19594,31 @@ "tensor": "model.language_model.layers.19.mlp.shared_expert.up_proj.weight" }, { - "bits": 6, + "bits": 8, "group_size": 32, "method": "affine", "metrics": { - "cosine_distance": 0.03022881489572491, + "cosine_distance": 0.021876116042958818, "decode_latency_cost": 0.0, - "hidden_state_error": 0.05374011537017762, - "long_context_loss": 0.020152543263816602, - "mtp_acceptance_loss": 0.005374011537017762, - "output_kl": 0.06717514421272203, + "hidden_state_error": 0.03889087296526012, + "long_context_loss": 0.014584077361972545, + "mtp_acceptance_loss": 0.003889087296526012, + "output_kl": 0.04861359120657515, "peak_memory_cost": 0.0, "prefill_latency_cost": 0.0, - "task_loss_delta": 0.043663843738269315, - "token_disagreement": 0.036946329316997116 + "task_loss_delta": 0.03159883428427385, + "token_disagreement": 0.026737475163616336 }, "module_path": "model.language_model.layers.19.mlp.shared_expert_gate", "outlier_strategy": "none", "parameters": 2048, - "predicted_loss": 0.036677628740146226, - "reason": "selected by marginal quality gain per storage bit", - "role": "expert", + "predicted_loss": 0.02654302079879003, + "reason": "protected router policy", + "role": "router", "scale_strategy": "group-affine", "strategy_metadata": { - "selected_from_candidates": 7, - "storage_bpw": 7.0 + "selected_from_candidates": 3, + "storage_bpw": 9.0 }, "tensor": "model.language_model.layers.19.mlp.shared_expert_gate.weight" }, @@ -20203,60 +20203,60 @@ "tensor": "model.language_model.layers.20.linear_attn.dt_bias" }, { - "bits": 6, - "group_size": 32, + "bits": 8, + "group_size": 64, "method": "affine", "metrics": { - "cosine_distance": 0.057275649276110355, + "cosine_distance": 0.02025, "decode_latency_cost": 0.0, - "hidden_state_error": 0.10182337649086284, - "long_context_loss": 0.15909902576697318, - "mtp_acceptance_loss": 0.010182337649086284, - "output_kl": 0.12727922061357855, + "hidden_state_error": 0.036, + "long_context_loss": 0.056249999999999994, + "mtp_acceptance_loss": 0.0036, + "output_kl": 0.045, "peak_memory_cost": 0.0, "prefill_latency_cost": 0.0, - "task_loss_delta": 0.08273149339882606, - "token_disagreement": 0.0700035713374682 + "task_loss_delta": 0.029249999999999998, + "token_disagreement": 0.02475 }, "module_path": "model.language_model.layers.20.linear_attn.in_proj_a", "outlier_strategy": "none", "parameters": 65536, - "predicted_loss": 0.07916767522164586, + "predicted_loss": 0.027989999999999998, "reason": "selected by marginal quality gain per storage bit", "role": "attention", "scale_strategy": "group-affine", "strategy_metadata": { "selected_from_candidates": 7, - "storage_bpw": 7.0 + "storage_bpw": 8.5 }, "tensor": "model.language_model.layers.20.linear_attn.in_proj_a.weight" }, { - "bits": 6, - "group_size": 32, + "bits": 8, + "group_size": 64, "method": "affine", "metrics": { - "cosine_distance": 0.057275649276110355, + "cosine_distance": 0.02025, "decode_latency_cost": 0.0, - "hidden_state_error": 0.10182337649086284, - "long_context_loss": 0.15909902576697318, - "mtp_acceptance_loss": 0.010182337649086284, - "output_kl": 0.12727922061357855, + "hidden_state_error": 0.036, + "long_context_loss": 0.056249999999999994, + "mtp_acceptance_loss": 0.0036, + "output_kl": 0.045, "peak_memory_cost": 0.0, "prefill_latency_cost": 0.0, - "task_loss_delta": 0.08273149339882606, - "token_disagreement": 0.0700035713374682 + "task_loss_delta": 0.029249999999999998, + "token_disagreement": 0.02475 }, "module_path": "model.language_model.layers.20.linear_attn.in_proj_b", "outlier_strategy": "none", "parameters": 65536, - "predicted_loss": 0.07916767522164586, + "predicted_loss": 0.027989999999999998, "reason": "selected by marginal quality gain per storage bit", "role": "attention", "scale_strategy": "group-affine", "strategy_metadata": { "selected_from_candidates": 7, - "storage_bpw": 7.0 + "storage_bpw": 8.5 }, "tensor": "model.language_model.layers.20.linear_attn.in_proj_b.weight" }, @@ -20406,31 +20406,31 @@ "tensor": "model.language_model.layers.20.mlp.shared_expert.up_proj.weight" }, { - "bits": 6, + "bits": 8, "group_size": 32, "method": "affine", "metrics": { - "cosine_distance": 0.03022881489572491, + "cosine_distance": 0.021876116042958818, "decode_latency_cost": 0.0, - "hidden_state_error": 0.05374011537017762, - "long_context_loss": 0.020152543263816602, - "mtp_acceptance_loss": 0.005374011537017762, - "output_kl": 0.06717514421272203, + "hidden_state_error": 0.03889087296526012, + "long_context_loss": 0.014584077361972545, + "mtp_acceptance_loss": 0.003889087296526012, + "output_kl": 0.04861359120657515, "peak_memory_cost": 0.0, "prefill_latency_cost": 0.0, - "task_loss_delta": 0.043663843738269315, - "token_disagreement": 0.036946329316997116 + "task_loss_delta": 0.03159883428427385, + "token_disagreement": 0.026737475163616336 }, "module_path": "model.language_model.layers.20.mlp.shared_expert_gate", "outlier_strategy": "none", "parameters": 2048, - "predicted_loss": 0.036677628740146226, - "reason": "selected by marginal quality gain per storage bit", - "role": "expert", + "predicted_loss": 0.02654302079879003, + "reason": "protected router policy", + "role": "router", "scale_strategy": "group-affine", "strategy_metadata": { - "selected_from_candidates": 7, - "storage_bpw": 7.0 + "selected_from_candidates": 3, + "storage_bpw": 9.0 }, "tensor": "model.language_model.layers.20.mlp.shared_expert_gate.weight" }, @@ -20580,60 +20580,60 @@ "tensor": "model.language_model.layers.21.linear_attn.dt_bias" }, { - "bits": 6, - "group_size": 32, + "bits": 8, + "group_size": 64, "method": "affine", "metrics": { - "cosine_distance": 0.057275649276110355, + "cosine_distance": 0.02025, "decode_latency_cost": 0.0, - "hidden_state_error": 0.10182337649086284, - "long_context_loss": 0.15909902576697318, - "mtp_acceptance_loss": 0.010182337649086284, - "output_kl": 0.12727922061357855, + "hidden_state_error": 0.036, + "long_context_loss": 0.056249999999999994, + "mtp_acceptance_loss": 0.0036, + "output_kl": 0.045, "peak_memory_cost": 0.0, "prefill_latency_cost": 0.0, - "task_loss_delta": 0.08273149339882606, - "token_disagreement": 0.0700035713374682 + "task_loss_delta": 0.029249999999999998, + "token_disagreement": 0.02475 }, "module_path": "model.language_model.layers.21.linear_attn.in_proj_a", "outlier_strategy": "none", "parameters": 65536, - "predicted_loss": 0.07916767522164586, + "predicted_loss": 0.027989999999999998, "reason": "selected by marginal quality gain per storage bit", "role": "attention", "scale_strategy": "group-affine", "strategy_metadata": { "selected_from_candidates": 7, - "storage_bpw": 7.0 + "storage_bpw": 8.5 }, "tensor": "model.language_model.layers.21.linear_attn.in_proj_a.weight" }, { - "bits": 6, - "group_size": 32, + "bits": 8, + "group_size": 64, "method": "affine", "metrics": { - "cosine_distance": 0.057275649276110355, + "cosine_distance": 0.02025, "decode_latency_cost": 0.0, - "hidden_state_error": 0.10182337649086284, - "long_context_loss": 0.15909902576697318, - "mtp_acceptance_loss": 0.010182337649086284, - "output_kl": 0.12727922061357855, + "hidden_state_error": 0.036, + "long_context_loss": 0.056249999999999994, + "mtp_acceptance_loss": 0.0036, + "output_kl": 0.045, "peak_memory_cost": 0.0, "prefill_latency_cost": 0.0, - "task_loss_delta": 0.08273149339882606, - "token_disagreement": 0.0700035713374682 + "task_loss_delta": 0.029249999999999998, + "token_disagreement": 0.02475 }, "module_path": "model.language_model.layers.21.linear_attn.in_proj_b", "outlier_strategy": "none", "parameters": 65536, - "predicted_loss": 0.07916767522164586, + "predicted_loss": 0.027989999999999998, "reason": "selected by marginal quality gain per storage bit", "role": "attention", "scale_strategy": "group-affine", "strategy_metadata": { "selected_from_candidates": 7, - "storage_bpw": 7.0 + "storage_bpw": 8.5 }, "tensor": "model.language_model.layers.21.linear_attn.in_proj_b.weight" }, @@ -20812,31 +20812,31 @@ "tensor": "model.language_model.layers.21.mlp.shared_expert.up_proj.weight" }, { - "bits": 6, + "bits": 8, "group_size": 32, "method": "affine", "metrics": { - "cosine_distance": 0.03022881489572491, + "cosine_distance": 0.021876116042958818, "decode_latency_cost": 0.0, - "hidden_state_error": 0.05374011537017762, - "long_context_loss": 0.020152543263816602, - "mtp_acceptance_loss": 0.005374011537017762, - "output_kl": 0.06717514421272203, + "hidden_state_error": 0.03889087296526012, + "long_context_loss": 0.014584077361972545, + "mtp_acceptance_loss": 0.003889087296526012, + "output_kl": 0.04861359120657515, "peak_memory_cost": 0.0, "prefill_latency_cost": 0.0, - "task_loss_delta": 0.043663843738269315, - "token_disagreement": 0.036946329316997116 + "task_loss_delta": 0.03159883428427385, + "token_disagreement": 0.026737475163616336 }, "module_path": "model.language_model.layers.21.mlp.shared_expert_gate", "outlier_strategy": "none", "parameters": 2048, - "predicted_loss": 0.036677628740146226, - "reason": "selected by marginal quality gain per storage bit", - "role": "expert", + "predicted_loss": 0.02654302079879003, + "reason": "protected router policy", + "role": "router", "scale_strategy": "group-affine", "strategy_metadata": { - "selected_from_candidates": 7, - "storage_bpw": 7.0 + "selected_from_candidates": 3, + "storage_bpw": 9.0 }, "tensor": "model.language_model.layers.21.mlp.shared_expert_gate.weight" }, @@ -20986,60 +20986,60 @@ "tensor": "model.language_model.layers.22.linear_attn.dt_bias" }, { - "bits": 6, - "group_size": 32, + "bits": 8, + "group_size": 64, "method": "affine", "metrics": { - "cosine_distance": 0.057275649276110355, + "cosine_distance": 0.02025, "decode_latency_cost": 0.0, - "hidden_state_error": 0.10182337649086284, - "long_context_loss": 0.15909902576697318, - "mtp_acceptance_loss": 0.010182337649086284, - "output_kl": 0.12727922061357855, + "hidden_state_error": 0.036, + "long_context_loss": 0.056249999999999994, + "mtp_acceptance_loss": 0.0036, + "output_kl": 0.045, "peak_memory_cost": 0.0, "prefill_latency_cost": 0.0, - "task_loss_delta": 0.08273149339882606, - "token_disagreement": 0.0700035713374682 + "task_loss_delta": 0.029249999999999998, + "token_disagreement": 0.02475 }, "module_path": "model.language_model.layers.22.linear_attn.in_proj_a", "outlier_strategy": "none", "parameters": 65536, - "predicted_loss": 0.07916767522164586, + "predicted_loss": 0.027989999999999998, "reason": "selected by marginal quality gain per storage bit", "role": "attention", "scale_strategy": "group-affine", "strategy_metadata": { "selected_from_candidates": 7, - "storage_bpw": 7.0 + "storage_bpw": 8.5 }, "tensor": "model.language_model.layers.22.linear_attn.in_proj_a.weight" }, { - "bits": 6, - "group_size": 32, + "bits": 8, + "group_size": 64, "method": "affine", "metrics": { - "cosine_distance": 0.057275649276110355, + "cosine_distance": 0.02025, "decode_latency_cost": 0.0, - "hidden_state_error": 0.10182337649086284, - "long_context_loss": 0.15909902576697318, - "mtp_acceptance_loss": 0.010182337649086284, - "output_kl": 0.12727922061357855, + "hidden_state_error": 0.036, + "long_context_loss": 0.056249999999999994, + "mtp_acceptance_loss": 0.0036, + "output_kl": 0.045, "peak_memory_cost": 0.0, "prefill_latency_cost": 0.0, - "task_loss_delta": 0.08273149339882606, - "token_disagreement": 0.0700035713374682 + "task_loss_delta": 0.029249999999999998, + "token_disagreement": 0.02475 }, "module_path": "model.language_model.layers.22.linear_attn.in_proj_b", "outlier_strategy": "none", "parameters": 65536, - "predicted_loss": 0.07916767522164586, + "predicted_loss": 0.027989999999999998, "reason": "selected by marginal quality gain per storage bit", "role": "attention", "scale_strategy": "group-affine", "strategy_metadata": { "selected_from_candidates": 7, - "storage_bpw": 7.0 + "storage_bpw": 8.5 }, "tensor": "model.language_model.layers.22.linear_attn.in_proj_b.weight" }, @@ -21218,31 +21218,31 @@ "tensor": "model.language_model.layers.22.mlp.shared_expert.up_proj.weight" }, { - "bits": 6, + "bits": 8, "group_size": 32, "method": "affine", "metrics": { - "cosine_distance": 0.03022881489572491, + "cosine_distance": 0.021876116042958818, "decode_latency_cost": 0.0, - "hidden_state_error": 0.05374011537017762, - "long_context_loss": 0.020152543263816602, - "mtp_acceptance_loss": 0.005374011537017762, - "output_kl": 0.06717514421272203, + "hidden_state_error": 0.03889087296526012, + "long_context_loss": 0.014584077361972545, + "mtp_acceptance_loss": 0.003889087296526012, + "output_kl": 0.04861359120657515, "peak_memory_cost": 0.0, "prefill_latency_cost": 0.0, - "task_loss_delta": 0.043663843738269315, - "token_disagreement": 0.036946329316997116 + "task_loss_delta": 0.03159883428427385, + "token_disagreement": 0.026737475163616336 }, "module_path": "model.language_model.layers.22.mlp.shared_expert_gate", "outlier_strategy": "none", "parameters": 2048, - "predicted_loss": 0.036677628740146226, - "reason": "selected by marginal quality gain per storage bit", - "role": "expert", + "predicted_loss": 0.02654302079879003, + "reason": "protected router policy", + "role": "router", "scale_strategy": "group-affine", "strategy_metadata": { - "selected_from_candidates": 7, - "storage_bpw": 7.0 + "selected_from_candidates": 3, + "storage_bpw": 9.0 }, "tensor": "model.language_model.layers.22.mlp.shared_expert_gate.weight" }, @@ -21334,31 +21334,31 @@ "tensor": "model.language_model.layers.23.mlp.experts.gate_up_proj" }, { - "bits": 6, - "group_size": 32, + "bits": 8, + "group_size": 64, "method": "affine", "metrics": { - "cosine_distance": 0.057275649276110355, + "cosine_distance": 0.02025, "decode_latency_cost": 0.0, - "hidden_state_error": 0.10182337649086284, - "long_context_loss": 0.15909902576697318, - "mtp_acceptance_loss": 0.010182337649086284, - "output_kl": 0.12727922061357855, + "hidden_state_error": 0.036, + "long_context_loss": 0.056249999999999994, + "mtp_acceptance_loss": 0.0036, + "output_kl": 0.045, "peak_memory_cost": 0.0, "prefill_latency_cost": 0.0, - "task_loss_delta": 0.08273149339882606, - "token_disagreement": 0.0700035713374682 + "task_loss_delta": 0.029249999999999998, + "token_disagreement": 0.02475 }, "module_path": "model.language_model.layers.23.self_attn.k_proj", "outlier_strategy": "none", "parameters": 1048576, - "predicted_loss": 0.07916767522164586, + "predicted_loss": 0.027989999999999998, "reason": "selected by marginal quality gain per storage bit", "role": "attention", "scale_strategy": "group-affine", "strategy_metadata": { "selected_from_candidates": 7, - "storage_bpw": 7.0 + "storage_bpw": 8.5 }, "tensor": "model.language_model.layers.23.self_attn.k_proj.weight" }, @@ -21421,31 +21421,31 @@ "tensor": "model.language_model.layers.23.self_attn.q_proj.weight" }, { - "bits": 6, - "group_size": 32, + "bits": 8, + "group_size": 64, "method": "affine", "metrics": { - "cosine_distance": 0.057275649276110355, + "cosine_distance": 0.02025, "decode_latency_cost": 0.0, - "hidden_state_error": 0.10182337649086284, - "long_context_loss": 0.15909902576697318, - "mtp_acceptance_loss": 0.010182337649086284, - "output_kl": 0.12727922061357855, + "hidden_state_error": 0.036, + "long_context_loss": 0.056249999999999994, + "mtp_acceptance_loss": 0.0036, + "output_kl": 0.045, "peak_memory_cost": 0.0, "prefill_latency_cost": 0.0, - "task_loss_delta": 0.08273149339882606, - "token_disagreement": 0.0700035713374682 + "task_loss_delta": 0.029249999999999998, + "token_disagreement": 0.02475 }, "module_path": "model.language_model.layers.23.self_attn.v_proj", "outlier_strategy": "none", "parameters": 1048576, - "predicted_loss": 0.07916767522164586, + "predicted_loss": 0.027989999999999998, "reason": "selected by marginal quality gain per storage bit", "role": "attention", "scale_strategy": "group-affine", "strategy_metadata": { "selected_from_candidates": 7, - "storage_bpw": 7.0 + "storage_bpw": 8.5 }, "tensor": "model.language_model.layers.23.self_attn.v_proj.weight" }, @@ -21856,31 +21856,31 @@ "tensor": "model.language_model.layers.23.mlp.shared_expert.up_proj.weight" }, { - "bits": 6, + "bits": 8, "group_size": 32, "method": "affine", "metrics": { - "cosine_distance": 0.03022881489572491, + "cosine_distance": 0.021876116042958818, "decode_latency_cost": 0.0, - "hidden_state_error": 0.05374011537017762, - "long_context_loss": 0.020152543263816602, - "mtp_acceptance_loss": 0.005374011537017762, - "output_kl": 0.06717514421272203, + "hidden_state_error": 0.03889087296526012, + "long_context_loss": 0.014584077361972545, + "mtp_acceptance_loss": 0.003889087296526012, + "output_kl": 0.04861359120657515, "peak_memory_cost": 0.0, "prefill_latency_cost": 0.0, - "task_loss_delta": 0.043663843738269315, - "token_disagreement": 0.036946329316997116 + "task_loss_delta": 0.03159883428427385, + "token_disagreement": 0.026737475163616336 }, "module_path": "model.language_model.layers.23.mlp.shared_expert_gate", "outlier_strategy": "none", "parameters": 2048, - "predicted_loss": 0.036677628740146226, - "reason": "selected by marginal quality gain per storage bit", - "role": "expert", + "predicted_loss": 0.02654302079879003, + "reason": "protected router policy", + "role": "router", "scale_strategy": "group-affine", "strategy_metadata": { - "selected_from_candidates": 7, - "storage_bpw": 7.0 + "selected_from_candidates": 3, + "storage_bpw": 9.0 }, "tensor": "model.language_model.layers.23.mlp.shared_expert_gate.weight" }, @@ -22088,60 +22088,60 @@ "tensor": "model.language_model.layers.24.linear_attn.dt_bias" }, { - "bits": 6, - "group_size": 32, + "bits": 8, + "group_size": 64, "method": "affine", "metrics": { - "cosine_distance": 0.057275649276110355, + "cosine_distance": 0.02025, "decode_latency_cost": 0.0, - "hidden_state_error": 0.10182337649086284, - "long_context_loss": 0.15909902576697318, - "mtp_acceptance_loss": 0.010182337649086284, - "output_kl": 0.12727922061357855, + "hidden_state_error": 0.036, + "long_context_loss": 0.056249999999999994, + "mtp_acceptance_loss": 0.0036, + "output_kl": 0.045, "peak_memory_cost": 0.0, "prefill_latency_cost": 0.0, - "task_loss_delta": 0.08273149339882606, - "token_disagreement": 0.0700035713374682 + "task_loss_delta": 0.029249999999999998, + "token_disagreement": 0.02475 }, "module_path": "model.language_model.layers.24.linear_attn.in_proj_a", "outlier_strategy": "none", "parameters": 65536, - "predicted_loss": 0.07916767522164586, + "predicted_loss": 0.027989999999999998, "reason": "selected by marginal quality gain per storage bit", "role": "attention", "scale_strategy": "group-affine", "strategy_metadata": { "selected_from_candidates": 7, - "storage_bpw": 7.0 + "storage_bpw": 8.5 }, "tensor": "model.language_model.layers.24.linear_attn.in_proj_a.weight" }, { - "bits": 6, - "group_size": 32, + "bits": 8, + "group_size": 64, "method": "affine", "metrics": { - "cosine_distance": 0.057275649276110355, + "cosine_distance": 0.02025, "decode_latency_cost": 0.0, - "hidden_state_error": 0.10182337649086284, - "long_context_loss": 0.15909902576697318, - "mtp_acceptance_loss": 0.010182337649086284, - "output_kl": 0.12727922061357855, + "hidden_state_error": 0.036, + "long_context_loss": 0.056249999999999994, + "mtp_acceptance_loss": 0.0036, + "output_kl": 0.045, "peak_memory_cost": 0.0, "prefill_latency_cost": 0.0, - "task_loss_delta": 0.08273149339882606, - "token_disagreement": 0.0700035713374682 + "task_loss_delta": 0.029249999999999998, + "token_disagreement": 0.02475 }, "module_path": "model.language_model.layers.24.linear_attn.in_proj_b", "outlier_strategy": "none", "parameters": 65536, - "predicted_loss": 0.07916767522164586, + "predicted_loss": 0.027989999999999998, "reason": "selected by marginal quality gain per storage bit", "role": "attention", "scale_strategy": "group-affine", "strategy_metadata": { "selected_from_candidates": 7, - "storage_bpw": 7.0 + "storage_bpw": 8.5 }, "tensor": "model.language_model.layers.24.linear_attn.in_proj_b.weight" }, @@ -22291,31 +22291,31 @@ "tensor": "model.language_model.layers.24.mlp.shared_expert.up_proj.weight" }, { - "bits": 6, + "bits": 8, "group_size": 32, "method": "affine", "metrics": { - "cosine_distance": 0.03022881489572491, + "cosine_distance": 0.021876116042958818, "decode_latency_cost": 0.0, - "hidden_state_error": 0.05374011537017762, - "long_context_loss": 0.020152543263816602, - "mtp_acceptance_loss": 0.005374011537017762, - "output_kl": 0.06717514421272203, + "hidden_state_error": 0.03889087296526012, + "long_context_loss": 0.014584077361972545, + "mtp_acceptance_loss": 0.003889087296526012, + "output_kl": 0.04861359120657515, "peak_memory_cost": 0.0, "prefill_latency_cost": 0.0, - "task_loss_delta": 0.043663843738269315, - "token_disagreement": 0.036946329316997116 + "task_loss_delta": 0.03159883428427385, + "token_disagreement": 0.026737475163616336 }, "module_path": "model.language_model.layers.24.mlp.shared_expert_gate", "outlier_strategy": "none", "parameters": 2048, - "predicted_loss": 0.036677628740146226, - "reason": "selected by marginal quality gain per storage bit", - "role": "expert", + "predicted_loss": 0.02654302079879003, + "reason": "protected router policy", + "role": "router", "scale_strategy": "group-affine", "strategy_metadata": { - "selected_from_candidates": 7, - "storage_bpw": 7.0 + "selected_from_candidates": 3, + "storage_bpw": 9.0 }, "tensor": "model.language_model.layers.24.mlp.shared_expert_gate.weight" }, @@ -22465,60 +22465,60 @@ "tensor": "model.language_model.layers.25.linear_attn.dt_bias" }, { - "bits": 6, - "group_size": 32, + "bits": 8, + "group_size": 64, "method": "affine", "metrics": { - "cosine_distance": 0.057275649276110355, + "cosine_distance": 0.02025, "decode_latency_cost": 0.0, - "hidden_state_error": 0.10182337649086284, - "long_context_loss": 0.15909902576697318, - "mtp_acceptance_loss": 0.010182337649086284, - "output_kl": 0.12727922061357855, + "hidden_state_error": 0.036, + "long_context_loss": 0.056249999999999994, + "mtp_acceptance_loss": 0.0036, + "output_kl": 0.045, "peak_memory_cost": 0.0, "prefill_latency_cost": 0.0, - "task_loss_delta": 0.08273149339882606, - "token_disagreement": 0.0700035713374682 + "task_loss_delta": 0.029249999999999998, + "token_disagreement": 0.02475 }, "module_path": "model.language_model.layers.25.linear_attn.in_proj_a", "outlier_strategy": "none", "parameters": 65536, - "predicted_loss": 0.07916767522164586, + "predicted_loss": 0.027989999999999998, "reason": "selected by marginal quality gain per storage bit", "role": "attention", "scale_strategy": "group-affine", "strategy_metadata": { "selected_from_candidates": 7, - "storage_bpw": 7.0 + "storage_bpw": 8.5 }, "tensor": "model.language_model.layers.25.linear_attn.in_proj_a.weight" }, { - "bits": 6, - "group_size": 32, + "bits": 8, + "group_size": 64, "method": "affine", "metrics": { - "cosine_distance": 0.057275649276110355, + "cosine_distance": 0.02025, "decode_latency_cost": 0.0, - "hidden_state_error": 0.10182337649086284, - "long_context_loss": 0.15909902576697318, - "mtp_acceptance_loss": 0.010182337649086284, - "output_kl": 0.12727922061357855, + "hidden_state_error": 0.036, + "long_context_loss": 0.056249999999999994, + "mtp_acceptance_loss": 0.0036, + "output_kl": 0.045, "peak_memory_cost": 0.0, "prefill_latency_cost": 0.0, - "task_loss_delta": 0.08273149339882606, - "token_disagreement": 0.0700035713374682 + "task_loss_delta": 0.029249999999999998, + "token_disagreement": 0.02475 }, "module_path": "model.language_model.layers.25.linear_attn.in_proj_b", "outlier_strategy": "none", "parameters": 65536, - "predicted_loss": 0.07916767522164586, + "predicted_loss": 0.027989999999999998, "reason": "selected by marginal quality gain per storage bit", "role": "attention", "scale_strategy": "group-affine", "strategy_metadata": { "selected_from_candidates": 7, - "storage_bpw": 7.0 + "storage_bpw": 8.5 }, "tensor": "model.language_model.layers.25.linear_attn.in_proj_b.weight" }, @@ -22697,31 +22697,31 @@ "tensor": "model.language_model.layers.25.mlp.shared_expert.up_proj.weight" }, { - "bits": 6, + "bits": 8, "group_size": 32, "method": "affine", "metrics": { - "cosine_distance": 0.03022881489572491, + "cosine_distance": 0.021876116042958818, "decode_latency_cost": 0.0, - "hidden_state_error": 0.05374011537017762, - "long_context_loss": 0.020152543263816602, - "mtp_acceptance_loss": 0.005374011537017762, - "output_kl": 0.06717514421272203, + "hidden_state_error": 0.03889087296526012, + "long_context_loss": 0.014584077361972545, + "mtp_acceptance_loss": 0.003889087296526012, + "output_kl": 0.04861359120657515, "peak_memory_cost": 0.0, "prefill_latency_cost": 0.0, - "task_loss_delta": 0.043663843738269315, - "token_disagreement": 0.036946329316997116 + "task_loss_delta": 0.03159883428427385, + "token_disagreement": 0.026737475163616336 }, "module_path": "model.language_model.layers.25.mlp.shared_expert_gate", "outlier_strategy": "none", "parameters": 2048, - "predicted_loss": 0.036677628740146226, - "reason": "selected by marginal quality gain per storage bit", - "role": "expert", + "predicted_loss": 0.02654302079879003, + "reason": "protected router policy", + "role": "router", "scale_strategy": "group-affine", "strategy_metadata": { - "selected_from_candidates": 7, - "storage_bpw": 7.0 + "selected_from_candidates": 3, + "storage_bpw": 9.0 }, "tensor": "model.language_model.layers.25.mlp.shared_expert_gate.weight" }, @@ -22900,31 +22900,31 @@ "tensor": "model.language_model.layers.27.mlp.experts.gate_up_proj" }, { - "bits": 6, - "group_size": 32, + "bits": 8, + "group_size": 64, "method": "affine", "metrics": { - "cosine_distance": 0.057275649276110355, + "cosine_distance": 0.02025, "decode_latency_cost": 0.0, - "hidden_state_error": 0.10182337649086284, - "long_context_loss": 0.15909902576697318, - "mtp_acceptance_loss": 0.010182337649086284, - "output_kl": 0.12727922061357855, + "hidden_state_error": 0.036, + "long_context_loss": 0.056249999999999994, + "mtp_acceptance_loss": 0.0036, + "output_kl": 0.045, "peak_memory_cost": 0.0, "prefill_latency_cost": 0.0, - "task_loss_delta": 0.08273149339882606, - "token_disagreement": 0.0700035713374682 + "task_loss_delta": 0.029249999999999998, + "token_disagreement": 0.02475 }, "module_path": "model.language_model.layers.27.self_attn.k_proj", "outlier_strategy": "none", "parameters": 1048576, - "predicted_loss": 0.07916767522164586, + "predicted_loss": 0.027989999999999998, "reason": "selected by marginal quality gain per storage bit", "role": "attention", "scale_strategy": "group-affine", "strategy_metadata": { "selected_from_candidates": 7, - "storage_bpw": 7.0 + "storage_bpw": 8.5 }, "tensor": "model.language_model.layers.27.self_attn.k_proj.weight" }, @@ -22987,31 +22987,31 @@ "tensor": "model.language_model.layers.27.self_attn.q_proj.weight" }, { - "bits": 6, - "group_size": 32, + "bits": 8, + "group_size": 64, "method": "affine", "metrics": { - "cosine_distance": 0.057275649276110355, + "cosine_distance": 0.02025, "decode_latency_cost": 0.0, - "hidden_state_error": 0.10182337649086284, - "long_context_loss": 0.15909902576697318, - "mtp_acceptance_loss": 0.010182337649086284, - "output_kl": 0.12727922061357855, + "hidden_state_error": 0.036, + "long_context_loss": 0.056249999999999994, + "mtp_acceptance_loss": 0.0036, + "output_kl": 0.045, "peak_memory_cost": 0.0, "prefill_latency_cost": 0.0, - "task_loss_delta": 0.08273149339882606, - "token_disagreement": 0.0700035713374682 + "task_loss_delta": 0.029249999999999998, + "token_disagreement": 0.02475 }, "module_path": "model.language_model.layers.27.self_attn.v_proj", "outlier_strategy": "none", "parameters": 1048576, - "predicted_loss": 0.07916767522164586, + "predicted_loss": 0.027989999999999998, "reason": "selected by marginal quality gain per storage bit", "role": "attention", "scale_strategy": "group-affine", "strategy_metadata": { "selected_from_candidates": 7, - "storage_bpw": 7.0 + "storage_bpw": 8.5 }, "tensor": "model.language_model.layers.27.self_attn.v_proj.weight" }, @@ -23277,60 +23277,60 @@ "tensor": "model.language_model.layers.26.linear_attn.dt_bias" }, { - "bits": 6, - "group_size": 32, + "bits": 8, + "group_size": 64, "method": "affine", "metrics": { - "cosine_distance": 0.057275649276110355, + "cosine_distance": 0.02025, "decode_latency_cost": 0.0, - "hidden_state_error": 0.10182337649086284, - "long_context_loss": 0.15909902576697318, - "mtp_acceptance_loss": 0.010182337649086284, - "output_kl": 0.12727922061357855, + "hidden_state_error": 0.036, + "long_context_loss": 0.056249999999999994, + "mtp_acceptance_loss": 0.0036, + "output_kl": 0.045, "peak_memory_cost": 0.0, "prefill_latency_cost": 0.0, - "task_loss_delta": 0.08273149339882606, - "token_disagreement": 0.0700035713374682 + "task_loss_delta": 0.029249999999999998, + "token_disagreement": 0.02475 }, "module_path": "model.language_model.layers.26.linear_attn.in_proj_a", "outlier_strategy": "none", "parameters": 65536, - "predicted_loss": 0.07916767522164586, + "predicted_loss": 0.027989999999999998, "reason": "selected by marginal quality gain per storage bit", "role": "attention", "scale_strategy": "group-affine", "strategy_metadata": { "selected_from_candidates": 7, - "storage_bpw": 7.0 + "storage_bpw": 8.5 }, "tensor": "model.language_model.layers.26.linear_attn.in_proj_a.weight" }, { - "bits": 6, - "group_size": 32, + "bits": 8, + "group_size": 64, "method": "affine", "metrics": { - "cosine_distance": 0.057275649276110355, + "cosine_distance": 0.02025, "decode_latency_cost": 0.0, - "hidden_state_error": 0.10182337649086284, - "long_context_loss": 0.15909902576697318, - "mtp_acceptance_loss": 0.010182337649086284, - "output_kl": 0.12727922061357855, + "hidden_state_error": 0.036, + "long_context_loss": 0.056249999999999994, + "mtp_acceptance_loss": 0.0036, + "output_kl": 0.045, "peak_memory_cost": 0.0, "prefill_latency_cost": 0.0, - "task_loss_delta": 0.08273149339882606, - "token_disagreement": 0.0700035713374682 + "task_loss_delta": 0.029249999999999998, + "token_disagreement": 0.02475 }, "module_path": "model.language_model.layers.26.linear_attn.in_proj_b", "outlier_strategy": "none", "parameters": 65536, - "predicted_loss": 0.07916767522164586, + "predicted_loss": 0.027989999999999998, "reason": "selected by marginal quality gain per storage bit", "role": "attention", "scale_strategy": "group-affine", "strategy_metadata": { "selected_from_candidates": 7, - "storage_bpw": 7.0 + "storage_bpw": 8.5 }, "tensor": "model.language_model.layers.26.linear_attn.in_proj_b.weight" }, @@ -23509,31 +23509,31 @@ "tensor": "model.language_model.layers.26.mlp.shared_expert.up_proj.weight" }, { - "bits": 6, + "bits": 8, "group_size": 32, "method": "affine", "metrics": { - "cosine_distance": 0.03022881489572491, + "cosine_distance": 0.021876116042958818, "decode_latency_cost": 0.0, - "hidden_state_error": 0.05374011537017762, - "long_context_loss": 0.020152543263816602, - "mtp_acceptance_loss": 0.005374011537017762, - "output_kl": 0.06717514421272203, + "hidden_state_error": 0.03889087296526012, + "long_context_loss": 0.014584077361972545, + "mtp_acceptance_loss": 0.003889087296526012, + "output_kl": 0.04861359120657515, "peak_memory_cost": 0.0, "prefill_latency_cost": 0.0, - "task_loss_delta": 0.043663843738269315, - "token_disagreement": 0.036946329316997116 + "task_loss_delta": 0.03159883428427385, + "token_disagreement": 0.026737475163616336 }, "module_path": "model.language_model.layers.26.mlp.shared_expert_gate", "outlier_strategy": "none", "parameters": 2048, - "predicted_loss": 0.036677628740146226, - "reason": "selected by marginal quality gain per storage bit", - "role": "expert", + "predicted_loss": 0.02654302079879003, + "reason": "protected router policy", + "role": "router", "scale_strategy": "group-affine", "strategy_metadata": { - "selected_from_candidates": 7, - "storage_bpw": 7.0 + "selected_from_candidates": 3, + "storage_bpw": 9.0 }, "tensor": "model.language_model.layers.26.mlp.shared_expert_gate.weight" }, @@ -23741,31 +23741,31 @@ "tensor": "model.language_model.layers.27.mlp.shared_expert.up_proj.weight" }, { - "bits": 6, + "bits": 8, "group_size": 32, "method": "affine", "metrics": { - "cosine_distance": 0.03022881489572491, + "cosine_distance": 0.021876116042958818, "decode_latency_cost": 0.0, - "hidden_state_error": 0.05374011537017762, - "long_context_loss": 0.020152543263816602, - "mtp_acceptance_loss": 0.005374011537017762, - "output_kl": 0.06717514421272203, + "hidden_state_error": 0.03889087296526012, + "long_context_loss": 0.014584077361972545, + "mtp_acceptance_loss": 0.003889087296526012, + "output_kl": 0.04861359120657515, "peak_memory_cost": 0.0, "prefill_latency_cost": 0.0, - "task_loss_delta": 0.043663843738269315, - "token_disagreement": 0.036946329316997116 + "task_loss_delta": 0.03159883428427385, + "token_disagreement": 0.026737475163616336 }, "module_path": "model.language_model.layers.27.mlp.shared_expert_gate", "outlier_strategy": "none", "parameters": 2048, - "predicted_loss": 0.036677628740146226, - "reason": "selected by marginal quality gain per storage bit", - "role": "expert", + "predicted_loss": 0.02654302079879003, + "reason": "protected router policy", + "role": "router", "scale_strategy": "group-affine", "strategy_metadata": { - "selected_from_candidates": 7, - "storage_bpw": 7.0 + "selected_from_candidates": 3, + "storage_bpw": 9.0 }, "tensor": "model.language_model.layers.27.mlp.shared_expert_gate.weight" }, @@ -23973,60 +23973,60 @@ "tensor": "model.language_model.layers.28.linear_attn.dt_bias" }, { - "bits": 6, - "group_size": 32, + "bits": 8, + "group_size": 64, "method": "affine", "metrics": { - "cosine_distance": 0.057275649276110355, + "cosine_distance": 0.02025, "decode_latency_cost": 0.0, - "hidden_state_error": 0.10182337649086284, - "long_context_loss": 0.15909902576697318, - "mtp_acceptance_loss": 0.010182337649086284, - "output_kl": 0.12727922061357855, + "hidden_state_error": 0.036, + "long_context_loss": 0.056249999999999994, + "mtp_acceptance_loss": 0.0036, + "output_kl": 0.045, "peak_memory_cost": 0.0, "prefill_latency_cost": 0.0, - "task_loss_delta": 0.08273149339882606, - "token_disagreement": 0.0700035713374682 + "task_loss_delta": 0.029249999999999998, + "token_disagreement": 0.02475 }, "module_path": "model.language_model.layers.28.linear_attn.in_proj_a", "outlier_strategy": "none", "parameters": 65536, - "predicted_loss": 0.07916767522164586, + "predicted_loss": 0.027989999999999998, "reason": "selected by marginal quality gain per storage bit", "role": "attention", "scale_strategy": "group-affine", "strategy_metadata": { "selected_from_candidates": 7, - "storage_bpw": 7.0 + "storage_bpw": 8.5 }, "tensor": "model.language_model.layers.28.linear_attn.in_proj_a.weight" }, { - "bits": 6, - "group_size": 32, + "bits": 8, + "group_size": 64, "method": "affine", "metrics": { - "cosine_distance": 0.057275649276110355, + "cosine_distance": 0.02025, "decode_latency_cost": 0.0, - "hidden_state_error": 0.10182337649086284, - "long_context_loss": 0.15909902576697318, - "mtp_acceptance_loss": 0.010182337649086284, - "output_kl": 0.12727922061357855, + "hidden_state_error": 0.036, + "long_context_loss": 0.056249999999999994, + "mtp_acceptance_loss": 0.0036, + "output_kl": 0.045, "peak_memory_cost": 0.0, "prefill_latency_cost": 0.0, - "task_loss_delta": 0.08273149339882606, - "token_disagreement": 0.0700035713374682 + "task_loss_delta": 0.029249999999999998, + "token_disagreement": 0.02475 }, "module_path": "model.language_model.layers.28.linear_attn.in_proj_b", "outlier_strategy": "none", "parameters": 65536, - "predicted_loss": 0.07916767522164586, + "predicted_loss": 0.027989999999999998, "reason": "selected by marginal quality gain per storage bit", "role": "attention", "scale_strategy": "group-affine", "strategy_metadata": { "selected_from_candidates": 7, - "storage_bpw": 7.0 + "storage_bpw": 8.5 }, "tensor": "model.language_model.layers.28.linear_attn.in_proj_b.weight" }, @@ -24176,31 +24176,31 @@ "tensor": "model.language_model.layers.28.mlp.shared_expert.up_proj.weight" }, { - "bits": 6, + "bits": 8, "group_size": 32, "method": "affine", "metrics": { - "cosine_distance": 0.03022881489572491, + "cosine_distance": 0.021876116042958818, "decode_latency_cost": 0.0, - "hidden_state_error": 0.05374011537017762, - "long_context_loss": 0.020152543263816602, - "mtp_acceptance_loss": 0.005374011537017762, - "output_kl": 0.06717514421272203, + "hidden_state_error": 0.03889087296526012, + "long_context_loss": 0.014584077361972545, + "mtp_acceptance_loss": 0.003889087296526012, + "output_kl": 0.04861359120657515, "peak_memory_cost": 0.0, "prefill_latency_cost": 0.0, - "task_loss_delta": 0.043663843738269315, - "token_disagreement": 0.036946329316997116 + "task_loss_delta": 0.03159883428427385, + "token_disagreement": 0.026737475163616336 }, "module_path": "model.language_model.layers.28.mlp.shared_expert_gate", "outlier_strategy": "none", "parameters": 2048, - "predicted_loss": 0.036677628740146226, - "reason": "selected by marginal quality gain per storage bit", - "role": "expert", + "predicted_loss": 0.02654302079879003, + "reason": "protected router policy", + "role": "router", "scale_strategy": "group-affine", "strategy_metadata": { - "selected_from_candidates": 7, - "storage_bpw": 7.0 + "selected_from_candidates": 3, + "storage_bpw": 9.0 }, "tensor": "model.language_model.layers.28.mlp.shared_expert_gate.weight" }, @@ -24524,31 +24524,31 @@ "tensor": "model.language_model.layers.31.mlp.experts.gate_up_proj" }, { - "bits": 6, - "group_size": 32, + "bits": 8, + "group_size": 64, "method": "affine", "metrics": { - "cosine_distance": 0.057275649276110355, + "cosine_distance": 0.02025, "decode_latency_cost": 0.0, - "hidden_state_error": 0.10182337649086284, - "long_context_loss": 0.15909902576697318, - "mtp_acceptance_loss": 0.010182337649086284, - "output_kl": 0.12727922061357855, + "hidden_state_error": 0.036, + "long_context_loss": 0.056249999999999994, + "mtp_acceptance_loss": 0.0036, + "output_kl": 0.045, "peak_memory_cost": 0.0, "prefill_latency_cost": 0.0, - "task_loss_delta": 0.08273149339882606, - "token_disagreement": 0.0700035713374682 + "task_loss_delta": 0.029249999999999998, + "token_disagreement": 0.02475 }, "module_path": "model.language_model.layers.31.self_attn.k_proj", "outlier_strategy": "none", "parameters": 1048576, - "predicted_loss": 0.07916767522164586, + "predicted_loss": 0.027989999999999998, "reason": "selected by marginal quality gain per storage bit", "role": "attention", "scale_strategy": "group-affine", "strategy_metadata": { "selected_from_candidates": 7, - "storage_bpw": 7.0 + "storage_bpw": 8.5 }, "tensor": "model.language_model.layers.31.self_attn.k_proj.weight" }, @@ -24611,31 +24611,31 @@ "tensor": "model.language_model.layers.31.self_attn.q_proj.weight" }, { - "bits": 6, - "group_size": 32, + "bits": 8, + "group_size": 64, "method": "affine", "metrics": { - "cosine_distance": 0.057275649276110355, + "cosine_distance": 0.02025, "decode_latency_cost": 0.0, - "hidden_state_error": 0.10182337649086284, - "long_context_loss": 0.15909902576697318, - "mtp_acceptance_loss": 0.010182337649086284, - "output_kl": 0.12727922061357855, + "hidden_state_error": 0.036, + "long_context_loss": 0.056249999999999994, + "mtp_acceptance_loss": 0.0036, + "output_kl": 0.045, "peak_memory_cost": 0.0, "prefill_latency_cost": 0.0, - "task_loss_delta": 0.08273149339882606, - "token_disagreement": 0.0700035713374682 + "task_loss_delta": 0.029249999999999998, + "token_disagreement": 0.02475 }, "module_path": "model.language_model.layers.31.self_attn.v_proj", "outlier_strategy": "none", "parameters": 1048576, - "predicted_loss": 0.07916767522164586, + "predicted_loss": 0.027989999999999998, "reason": "selected by marginal quality gain per storage bit", "role": "attention", "scale_strategy": "group-affine", "strategy_metadata": { "selected_from_candidates": 7, - "storage_bpw": 7.0 + "storage_bpw": 8.5 }, "tensor": "model.language_model.layers.31.self_attn.v_proj.weight" }, @@ -24756,60 +24756,60 @@ "tensor": "model.language_model.layers.29.linear_attn.dt_bias" }, { - "bits": 6, - "group_size": 32, + "bits": 8, + "group_size": 64, "method": "affine", "metrics": { - "cosine_distance": 0.057275649276110355, + "cosine_distance": 0.02025, "decode_latency_cost": 0.0, - "hidden_state_error": 0.10182337649086284, - "long_context_loss": 0.15909902576697318, - "mtp_acceptance_loss": 0.010182337649086284, - "output_kl": 0.12727922061357855, + "hidden_state_error": 0.036, + "long_context_loss": 0.056249999999999994, + "mtp_acceptance_loss": 0.0036, + "output_kl": 0.045, "peak_memory_cost": 0.0, "prefill_latency_cost": 0.0, - "task_loss_delta": 0.08273149339882606, - "token_disagreement": 0.0700035713374682 + "task_loss_delta": 0.029249999999999998, + "token_disagreement": 0.02475 }, "module_path": "model.language_model.layers.29.linear_attn.in_proj_a", "outlier_strategy": "none", "parameters": 65536, - "predicted_loss": 0.07916767522164586, + "predicted_loss": 0.027989999999999998, "reason": "selected by marginal quality gain per storage bit", "role": "attention", "scale_strategy": "group-affine", "strategy_metadata": { "selected_from_candidates": 7, - "storage_bpw": 7.0 + "storage_bpw": 8.5 }, "tensor": "model.language_model.layers.29.linear_attn.in_proj_a.weight" }, { - "bits": 6, - "group_size": 32, + "bits": 8, + "group_size": 64, "method": "affine", "metrics": { - "cosine_distance": 0.057275649276110355, + "cosine_distance": 0.02025, "decode_latency_cost": 0.0, - "hidden_state_error": 0.10182337649086284, - "long_context_loss": 0.15909902576697318, - "mtp_acceptance_loss": 0.010182337649086284, - "output_kl": 0.12727922061357855, + "hidden_state_error": 0.036, + "long_context_loss": 0.056249999999999994, + "mtp_acceptance_loss": 0.0036, + "output_kl": 0.045, "peak_memory_cost": 0.0, "prefill_latency_cost": 0.0, - "task_loss_delta": 0.08273149339882606, - "token_disagreement": 0.0700035713374682 + "task_loss_delta": 0.029249999999999998, + "token_disagreement": 0.02475 }, "module_path": "model.language_model.layers.29.linear_attn.in_proj_b", "outlier_strategy": "none", "parameters": 65536, - "predicted_loss": 0.07916767522164586, + "predicted_loss": 0.027989999999999998, "reason": "selected by marginal quality gain per storage bit", "role": "attention", "scale_strategy": "group-affine", "strategy_metadata": { "selected_from_candidates": 7, - "storage_bpw": 7.0 + "storage_bpw": 8.5 }, "tensor": "model.language_model.layers.29.linear_attn.in_proj_b.weight" }, @@ -24988,31 +24988,31 @@ "tensor": "model.language_model.layers.29.mlp.shared_expert.up_proj.weight" }, { - "bits": 6, + "bits": 8, "group_size": 32, "method": "affine", "metrics": { - "cosine_distance": 0.03022881489572491, + "cosine_distance": 0.021876116042958818, "decode_latency_cost": 0.0, - "hidden_state_error": 0.05374011537017762, - "long_context_loss": 0.020152543263816602, - "mtp_acceptance_loss": 0.005374011537017762, - "output_kl": 0.06717514421272203, + "hidden_state_error": 0.03889087296526012, + "long_context_loss": 0.014584077361972545, + "mtp_acceptance_loss": 0.003889087296526012, + "output_kl": 0.04861359120657515, "peak_memory_cost": 0.0, "prefill_latency_cost": 0.0, - "task_loss_delta": 0.043663843738269315, - "token_disagreement": 0.036946329316997116 + "task_loss_delta": 0.03159883428427385, + "token_disagreement": 0.026737475163616336 }, "module_path": "model.language_model.layers.29.mlp.shared_expert_gate", "outlier_strategy": "none", "parameters": 2048, - "predicted_loss": 0.036677628740146226, - "reason": "selected by marginal quality gain per storage bit", - "role": "expert", + "predicted_loss": 0.02654302079879003, + "reason": "protected router policy", + "role": "router", "scale_strategy": "group-affine", "strategy_metadata": { - "selected_from_candidates": 7, - "storage_bpw": 7.0 + "selected_from_candidates": 3, + "storage_bpw": 9.0 }, "tensor": "model.language_model.layers.29.mlp.shared_expert_gate.weight" }, @@ -25162,60 +25162,60 @@ "tensor": "model.language_model.layers.30.linear_attn.dt_bias" }, { - "bits": 6, - "group_size": 32, + "bits": 8, + "group_size": 64, "method": "affine", "metrics": { - "cosine_distance": 0.057275649276110355, + "cosine_distance": 0.02025, "decode_latency_cost": 0.0, - "hidden_state_error": 0.10182337649086284, - "long_context_loss": 0.15909902576697318, - "mtp_acceptance_loss": 0.010182337649086284, - "output_kl": 0.12727922061357855, + "hidden_state_error": 0.036, + "long_context_loss": 0.056249999999999994, + "mtp_acceptance_loss": 0.0036, + "output_kl": 0.045, "peak_memory_cost": 0.0, "prefill_latency_cost": 0.0, - "task_loss_delta": 0.08273149339882606, - "token_disagreement": 0.0700035713374682 + "task_loss_delta": 0.029249999999999998, + "token_disagreement": 0.02475 }, "module_path": "model.language_model.layers.30.linear_attn.in_proj_a", "outlier_strategy": "none", "parameters": 65536, - "predicted_loss": 0.07916767522164586, + "predicted_loss": 0.027989999999999998, "reason": "selected by marginal quality gain per storage bit", "role": "attention", "scale_strategy": "group-affine", "strategy_metadata": { "selected_from_candidates": 7, - "storage_bpw": 7.0 + "storage_bpw": 8.5 }, "tensor": "model.language_model.layers.30.linear_attn.in_proj_a.weight" }, { - "bits": 6, - "group_size": 32, + "bits": 8, + "group_size": 64, "method": "affine", "metrics": { - "cosine_distance": 0.057275649276110355, + "cosine_distance": 0.02025, "decode_latency_cost": 0.0, - "hidden_state_error": 0.10182337649086284, - "long_context_loss": 0.15909902576697318, - "mtp_acceptance_loss": 0.010182337649086284, - "output_kl": 0.12727922061357855, + "hidden_state_error": 0.036, + "long_context_loss": 0.056249999999999994, + "mtp_acceptance_loss": 0.0036, + "output_kl": 0.045, "peak_memory_cost": 0.0, "prefill_latency_cost": 0.0, - "task_loss_delta": 0.08273149339882606, - "token_disagreement": 0.0700035713374682 + "task_loss_delta": 0.029249999999999998, + "token_disagreement": 0.02475 }, "module_path": "model.language_model.layers.30.linear_attn.in_proj_b", "outlier_strategy": "none", "parameters": 65536, - "predicted_loss": 0.07916767522164586, + "predicted_loss": 0.027989999999999998, "reason": "selected by marginal quality gain per storage bit", "role": "attention", "scale_strategy": "group-affine", "strategy_metadata": { "selected_from_candidates": 7, - "storage_bpw": 7.0 + "storage_bpw": 8.5 }, "tensor": "model.language_model.layers.30.linear_attn.in_proj_b.weight" }, @@ -25394,31 +25394,31 @@ "tensor": "model.language_model.layers.30.mlp.shared_expert.up_proj.weight" }, { - "bits": 6, + "bits": 8, "group_size": 32, "method": "affine", "metrics": { - "cosine_distance": 0.03022881489572491, + "cosine_distance": 0.021876116042958818, "decode_latency_cost": 0.0, - "hidden_state_error": 0.05374011537017762, - "long_context_loss": 0.020152543263816602, - "mtp_acceptance_loss": 0.005374011537017762, - "output_kl": 0.06717514421272203, + "hidden_state_error": 0.03889087296526012, + "long_context_loss": 0.014584077361972545, + "mtp_acceptance_loss": 0.003889087296526012, + "output_kl": 0.04861359120657515, "peak_memory_cost": 0.0, "prefill_latency_cost": 0.0, - "task_loss_delta": 0.043663843738269315, - "token_disagreement": 0.036946329316997116 + "task_loss_delta": 0.03159883428427385, + "token_disagreement": 0.026737475163616336 }, "module_path": "model.language_model.layers.30.mlp.shared_expert_gate", "outlier_strategy": "none", "parameters": 2048, - "predicted_loss": 0.036677628740146226, - "reason": "selected by marginal quality gain per storage bit", - "role": "expert", + "predicted_loss": 0.02654302079879003, + "reason": "protected router policy", + "role": "router", "scale_strategy": "group-affine", "strategy_metadata": { - "selected_from_candidates": 7, - "storage_bpw": 7.0 + "selected_from_candidates": 3, + "storage_bpw": 9.0 }, "tensor": "model.language_model.layers.30.mlp.shared_expert_gate.weight" }, @@ -25597,31 +25597,31 @@ "tensor": "model.language_model.layers.31.mlp.shared_expert.up_proj.weight" }, { - "bits": 6, + "bits": 8, "group_size": 32, "method": "affine", "metrics": { - "cosine_distance": 0.03022881489572491, + "cosine_distance": 0.021876116042958818, "decode_latency_cost": 0.0, - "hidden_state_error": 0.05374011537017762, - "long_context_loss": 0.020152543263816602, - "mtp_acceptance_loss": 0.005374011537017762, - "output_kl": 0.06717514421272203, + "hidden_state_error": 0.03889087296526012, + "long_context_loss": 0.014584077361972545, + "mtp_acceptance_loss": 0.003889087296526012, + "output_kl": 0.04861359120657515, "peak_memory_cost": 0.0, "prefill_latency_cost": 0.0, - "task_loss_delta": 0.043663843738269315, - "token_disagreement": 0.036946329316997116 + "task_loss_delta": 0.03159883428427385, + "token_disagreement": 0.026737475163616336 }, "module_path": "model.language_model.layers.31.mlp.shared_expert_gate", "outlier_strategy": "none", "parameters": 2048, - "predicted_loss": 0.036677628740146226, - "reason": "selected by marginal quality gain per storage bit", - "role": "expert", + "predicted_loss": 0.02654302079879003, + "reason": "protected router policy", + "role": "router", "scale_strategy": "group-affine", "strategy_metadata": { - "selected_from_candidates": 7, - "storage_bpw": 7.0 + "selected_from_candidates": 3, + "storage_bpw": 9.0 }, "tensor": "model.language_model.layers.31.mlp.shared_expert_gate.weight" }, @@ -25829,60 +25829,60 @@ "tensor": "model.language_model.layers.32.linear_attn.dt_bias" }, { - "bits": 6, - "group_size": 32, + "bits": 8, + "group_size": 64, "method": "affine", "metrics": { - "cosine_distance": 0.057275649276110355, + "cosine_distance": 0.02025, "decode_latency_cost": 0.0, - "hidden_state_error": 0.10182337649086284, - "long_context_loss": 0.15909902576697318, - "mtp_acceptance_loss": 0.010182337649086284, - "output_kl": 0.12727922061357855, + "hidden_state_error": 0.036, + "long_context_loss": 0.056249999999999994, + "mtp_acceptance_loss": 0.0036, + "output_kl": 0.045, "peak_memory_cost": 0.0, "prefill_latency_cost": 0.0, - "task_loss_delta": 0.08273149339882606, - "token_disagreement": 0.0700035713374682 + "task_loss_delta": 0.029249999999999998, + "token_disagreement": 0.02475 }, "module_path": "model.language_model.layers.32.linear_attn.in_proj_a", "outlier_strategy": "none", "parameters": 65536, - "predicted_loss": 0.07916767522164586, + "predicted_loss": 0.027989999999999998, "reason": "selected by marginal quality gain per storage bit", "role": "attention", "scale_strategy": "group-affine", "strategy_metadata": { "selected_from_candidates": 7, - "storage_bpw": 7.0 + "storage_bpw": 8.5 }, "tensor": "model.language_model.layers.32.linear_attn.in_proj_a.weight" }, { - "bits": 6, - "group_size": 32, + "bits": 8, + "group_size": 64, "method": "affine", "metrics": { - "cosine_distance": 0.057275649276110355, + "cosine_distance": 0.02025, "decode_latency_cost": 0.0, - "hidden_state_error": 0.10182337649086284, - "long_context_loss": 0.15909902576697318, - "mtp_acceptance_loss": 0.010182337649086284, - "output_kl": 0.12727922061357855, + "hidden_state_error": 0.036, + "long_context_loss": 0.056249999999999994, + "mtp_acceptance_loss": 0.0036, + "output_kl": 0.045, "peak_memory_cost": 0.0, "prefill_latency_cost": 0.0, - "task_loss_delta": 0.08273149339882606, - "token_disagreement": 0.0700035713374682 + "task_loss_delta": 0.029249999999999998, + "token_disagreement": 0.02475 }, "module_path": "model.language_model.layers.32.linear_attn.in_proj_b", "outlier_strategy": "none", "parameters": 65536, - "predicted_loss": 0.07916767522164586, + "predicted_loss": 0.027989999999999998, "reason": "selected by marginal quality gain per storage bit", "role": "attention", "scale_strategy": "group-affine", "strategy_metadata": { "selected_from_candidates": 7, - "storage_bpw": 7.0 + "storage_bpw": 8.5 }, "tensor": "model.language_model.layers.32.linear_attn.in_proj_b.weight" }, @@ -26177,31 +26177,31 @@ "tensor": "model.language_model.layers.32.mlp.shared_expert.up_proj.weight" }, { - "bits": 6, + "bits": 8, "group_size": 32, "method": "affine", "metrics": { - "cosine_distance": 0.03022881489572491, + "cosine_distance": 0.021876116042958818, "decode_latency_cost": 0.0, - "hidden_state_error": 0.05374011537017762, - "long_context_loss": 0.020152543263816602, - "mtp_acceptance_loss": 0.005374011537017762, - "output_kl": 0.06717514421272203, + "hidden_state_error": 0.03889087296526012, + "long_context_loss": 0.014584077361972545, + "mtp_acceptance_loss": 0.003889087296526012, + "output_kl": 0.04861359120657515, "peak_memory_cost": 0.0, "prefill_latency_cost": 0.0, - "task_loss_delta": 0.043663843738269315, - "token_disagreement": 0.036946329316997116 + "task_loss_delta": 0.03159883428427385, + "token_disagreement": 0.026737475163616336 }, "module_path": "model.language_model.layers.32.mlp.shared_expert_gate", "outlier_strategy": "none", "parameters": 2048, - "predicted_loss": 0.036677628740146226, - "reason": "selected by marginal quality gain per storage bit", - "role": "expert", + "predicted_loss": 0.02654302079879003, + "reason": "protected router policy", + "role": "router", "scale_strategy": "group-affine", "strategy_metadata": { - "selected_from_candidates": 7, - "storage_bpw": 7.0 + "selected_from_candidates": 3, + "storage_bpw": 9.0 }, "tensor": "model.language_model.layers.32.mlp.shared_expert_gate.weight" }, @@ -26351,60 +26351,60 @@ "tensor": "model.language_model.layers.33.linear_attn.dt_bias" }, { - "bits": 6, - "group_size": 32, + "bits": 8, + "group_size": 64, "method": "affine", "metrics": { - "cosine_distance": 0.057275649276110355, + "cosine_distance": 0.02025, "decode_latency_cost": 0.0, - "hidden_state_error": 0.10182337649086284, - "long_context_loss": 0.15909902576697318, - "mtp_acceptance_loss": 0.010182337649086284, - "output_kl": 0.12727922061357855, + "hidden_state_error": 0.036, + "long_context_loss": 0.056249999999999994, + "mtp_acceptance_loss": 0.0036, + "output_kl": 0.045, "peak_memory_cost": 0.0, "prefill_latency_cost": 0.0, - "task_loss_delta": 0.08273149339882606, - "token_disagreement": 0.0700035713374682 + "task_loss_delta": 0.029249999999999998, + "token_disagreement": 0.02475 }, "module_path": "model.language_model.layers.33.linear_attn.in_proj_a", "outlier_strategy": "none", "parameters": 65536, - "predicted_loss": 0.07916767522164586, + "predicted_loss": 0.027989999999999998, "reason": "selected by marginal quality gain per storage bit", "role": "attention", "scale_strategy": "group-affine", "strategy_metadata": { "selected_from_candidates": 7, - "storage_bpw": 7.0 + "storage_bpw": 8.5 }, "tensor": "model.language_model.layers.33.linear_attn.in_proj_a.weight" }, { - "bits": 6, - "group_size": 32, + "bits": 8, + "group_size": 64, "method": "affine", "metrics": { - "cosine_distance": 0.057275649276110355, + "cosine_distance": 0.02025, "decode_latency_cost": 0.0, - "hidden_state_error": 0.10182337649086284, - "long_context_loss": 0.15909902576697318, - "mtp_acceptance_loss": 0.010182337649086284, - "output_kl": 0.12727922061357855, + "hidden_state_error": 0.036, + "long_context_loss": 0.056249999999999994, + "mtp_acceptance_loss": 0.0036, + "output_kl": 0.045, "peak_memory_cost": 0.0, "prefill_latency_cost": 0.0, - "task_loss_delta": 0.08273149339882606, - "token_disagreement": 0.0700035713374682 + "task_loss_delta": 0.029249999999999998, + "token_disagreement": 0.02475 }, "module_path": "model.language_model.layers.33.linear_attn.in_proj_b", "outlier_strategy": "none", "parameters": 65536, - "predicted_loss": 0.07916767522164586, + "predicted_loss": 0.027989999999999998, "reason": "selected by marginal quality gain per storage bit", "role": "attention", "scale_strategy": "group-affine", "strategy_metadata": { "selected_from_candidates": 7, - "storage_bpw": 7.0 + "storage_bpw": 8.5 }, "tensor": "model.language_model.layers.33.linear_attn.in_proj_b.weight" }, @@ -26699,31 +26699,31 @@ "tensor": "model.language_model.layers.33.mlp.shared_expert.up_proj.weight" }, { - "bits": 6, + "bits": 8, "group_size": 32, "method": "affine", "metrics": { - "cosine_distance": 0.03022881489572491, + "cosine_distance": 0.021876116042958818, "decode_latency_cost": 0.0, - "hidden_state_error": 0.05374011537017762, - "long_context_loss": 0.020152543263816602, - "mtp_acceptance_loss": 0.005374011537017762, - "output_kl": 0.06717514421272203, + "hidden_state_error": 0.03889087296526012, + "long_context_loss": 0.014584077361972545, + "mtp_acceptance_loss": 0.003889087296526012, + "output_kl": 0.04861359120657515, "peak_memory_cost": 0.0, "prefill_latency_cost": 0.0, - "task_loss_delta": 0.043663843738269315, - "token_disagreement": 0.036946329316997116 + "task_loss_delta": 0.03159883428427385, + "token_disagreement": 0.026737475163616336 }, "module_path": "model.language_model.layers.33.mlp.shared_expert_gate", "outlier_strategy": "none", "parameters": 2048, - "predicted_loss": 0.036677628740146226, - "reason": "selected by marginal quality gain per storage bit", - "role": "expert", + "predicted_loss": 0.02654302079879003, + "reason": "protected router policy", + "role": "router", "scale_strategy": "group-affine", "strategy_metadata": { - "selected_from_candidates": 7, - "storage_bpw": 7.0 + "selected_from_candidates": 3, + "storage_bpw": 9.0 }, "tensor": "model.language_model.layers.33.mlp.shared_expert_gate.weight" }, @@ -26873,31 +26873,31 @@ "tensor": "model.language_model.layers.34.linear_attn.dt_bias" }, { - "bits": 6, - "group_size": 32, + "bits": 8, + "group_size": 64, "method": "affine", "metrics": { - "cosine_distance": 0.057275649276110355, + "cosine_distance": 0.02025, "decode_latency_cost": 0.0, - "hidden_state_error": 0.10182337649086284, - "long_context_loss": 0.15909902576697318, - "mtp_acceptance_loss": 0.010182337649086284, - "output_kl": 0.12727922061357855, + "hidden_state_error": 0.036, + "long_context_loss": 0.056249999999999994, + "mtp_acceptance_loss": 0.0036, + "output_kl": 0.045, "peak_memory_cost": 0.0, "prefill_latency_cost": 0.0, - "task_loss_delta": 0.08273149339882606, - "token_disagreement": 0.0700035713374682 + "task_loss_delta": 0.029249999999999998, + "token_disagreement": 0.02475 }, "module_path": "model.language_model.layers.34.linear_attn.in_proj_a", "outlier_strategy": "none", "parameters": 65536, - "predicted_loss": 0.07916767522164586, + "predicted_loss": 0.027989999999999998, "reason": "selected by marginal quality gain per storage bit", "role": "attention", "scale_strategy": "group-affine", "strategy_metadata": { "selected_from_candidates": 7, - "storage_bpw": 7.0 + "storage_bpw": 8.5 }, "tensor": "model.language_model.layers.34.linear_attn.in_proj_a.weight" }, @@ -26960,31 +26960,31 @@ "tensor": "model.language_model.layers.34.linear_attn.in_proj_qkv.weight" }, { - "bits": 8, - "group_size": 64, + "bits": 6, + "group_size": 32, "method": "affine", "metrics": { - "cosine_distance": 0.02025, + "cosine_distance": 0.057275649276110355, "decode_latency_cost": 0.0, - "hidden_state_error": 0.036, - "long_context_loss": 0.056249999999999994, - "mtp_acceptance_loss": 0.0036, - "output_kl": 0.045, + "hidden_state_error": 0.10182337649086284, + "long_context_loss": 0.15909902576697318, + "mtp_acceptance_loss": 0.010182337649086284, + "output_kl": 0.12727922061357855, "peak_memory_cost": 0.0, "prefill_latency_cost": 0.0, - "task_loss_delta": 0.029249999999999998, - "token_disagreement": 0.02475 + "task_loss_delta": 0.08273149339882606, + "token_disagreement": 0.0700035713374682 }, "module_path": "model.language_model.layers.34.linear_attn.in_proj_z", "outlier_strategy": "none", "parameters": 8388608, - "predicted_loss": 0.027989999999999998, + "predicted_loss": 0.07916767522164586, "reason": "selected by marginal quality gain per storage bit", "role": "attention", "scale_strategy": "group-affine", "strategy_metadata": { "selected_from_candidates": 7, - "storage_bpw": 8.5 + "storage_bpw": 7.0 }, "tensor": "model.language_model.layers.34.linear_attn.in_proj_z.weight" }, @@ -27221,31 +27221,31 @@ "tensor": "model.language_model.layers.34.mlp.shared_expert.up_proj.weight" }, { - "bits": 6, + "bits": 8, "group_size": 32, "method": "affine", "metrics": { - "cosine_distance": 0.03022881489572491, + "cosine_distance": 0.021876116042958818, "decode_latency_cost": 0.0, - "hidden_state_error": 0.05374011537017762, - "long_context_loss": 0.020152543263816602, - "mtp_acceptance_loss": 0.005374011537017762, - "output_kl": 0.06717514421272203, + "hidden_state_error": 0.03889087296526012, + "long_context_loss": 0.014584077361972545, + "mtp_acceptance_loss": 0.003889087296526012, + "output_kl": 0.04861359120657515, "peak_memory_cost": 0.0, "prefill_latency_cost": 0.0, - "task_loss_delta": 0.043663843738269315, - "token_disagreement": 0.036946329316997116 + "task_loss_delta": 0.03159883428427385, + "token_disagreement": 0.026737475163616336 }, "module_path": "model.language_model.layers.34.mlp.shared_expert_gate", "outlier_strategy": "none", "parameters": 2048, - "predicted_loss": 0.036677628740146226, - "reason": "selected by marginal quality gain per storage bit", - "role": "expert", + "predicted_loss": 0.02654302079879003, + "reason": "protected router policy", + "role": "router", "scale_strategy": "group-affine", "strategy_metadata": { - "selected_from_candidates": 7, - "storage_bpw": 7.0 + "selected_from_candidates": 3, + "storage_bpw": 9.0 }, "tensor": "model.language_model.layers.34.mlp.shared_expert_gate.weight" }, @@ -27482,31 +27482,31 @@ "tensor": "model.language_model.layers.35.mlp.shared_expert.up_proj.weight" }, { - "bits": 6, + "bits": 8, "group_size": 32, "method": "affine", "metrics": { - "cosine_distance": 0.03022881489572491, + "cosine_distance": 0.021876116042958818, "decode_latency_cost": 0.0, - "hidden_state_error": 0.05374011537017762, - "long_context_loss": 0.020152543263816602, - "mtp_acceptance_loss": 0.005374011537017762, - "output_kl": 0.06717514421272203, + "hidden_state_error": 0.03889087296526012, + "long_context_loss": 0.014584077361972545, + "mtp_acceptance_loss": 0.003889087296526012, + "output_kl": 0.04861359120657515, "peak_memory_cost": 0.0, "prefill_latency_cost": 0.0, - "task_loss_delta": 0.043663843738269315, - "token_disagreement": 0.036946329316997116 + "task_loss_delta": 0.03159883428427385, + "token_disagreement": 0.026737475163616336 }, "module_path": "model.language_model.layers.35.mlp.shared_expert_gate", "outlier_strategy": "none", "parameters": 2048, - "predicted_loss": 0.036677628740146226, - "reason": "selected by marginal quality gain per storage bit", - "role": "expert", + "predicted_loss": 0.02654302079879003, + "reason": "protected router policy", + "role": "router", "scale_strategy": "group-affine", "strategy_metadata": { - "selected_from_candidates": 7, - "storage_bpw": 7.0 + "selected_from_candidates": 3, + "storage_bpw": 9.0 }, "tensor": "model.language_model.layers.35.mlp.shared_expert_gate.weight" }, @@ -28178,31 +28178,31 @@ "tensor": "model.language_model.layers.36.mlp.shared_expert.up_proj.weight" }, { - "bits": 6, + "bits": 8, "group_size": 32, "method": "affine", "metrics": { - "cosine_distance": 0.03022881489572491, + "cosine_distance": 0.021876116042958818, "decode_latency_cost": 0.0, - "hidden_state_error": 0.05374011537017762, - "long_context_loss": 0.020152543263816602, - "mtp_acceptance_loss": 0.005374011537017762, - "output_kl": 0.06717514421272203, + "hidden_state_error": 0.03889087296526012, + "long_context_loss": 0.014584077361972545, + "mtp_acceptance_loss": 0.003889087296526012, + "output_kl": 0.04861359120657515, "peak_memory_cost": 0.0, "prefill_latency_cost": 0.0, - "task_loss_delta": 0.043663843738269315, - "token_disagreement": 0.036946329316997116 + "task_loss_delta": 0.03159883428427385, + "token_disagreement": 0.026737475163616336 }, "module_path": "model.language_model.layers.36.mlp.shared_expert_gate", "outlier_strategy": "none", "parameters": 2048, - "predicted_loss": 0.036677628740146226, - "reason": "selected by marginal quality gain per storage bit", - "role": "expert", + "predicted_loss": 0.02654302079879003, + "reason": "protected router policy", + "role": "router", "scale_strategy": "group-affine", "strategy_metadata": { - "selected_from_candidates": 7, - "storage_bpw": 7.0 + "selected_from_candidates": 3, + "storage_bpw": 9.0 }, "tensor": "model.language_model.layers.36.mlp.shared_expert_gate.weight" }, @@ -28700,31 +28700,31 @@ "tensor": "model.language_model.layers.37.mlp.shared_expert.up_proj.weight" }, { - "bits": 6, + "bits": 8, "group_size": 32, "method": "affine", "metrics": { - "cosine_distance": 0.03022881489572491, + "cosine_distance": 0.021876116042958818, "decode_latency_cost": 0.0, - "hidden_state_error": 0.05374011537017762, - "long_context_loss": 0.020152543263816602, - "mtp_acceptance_loss": 0.005374011537017762, - "output_kl": 0.06717514421272203, + "hidden_state_error": 0.03889087296526012, + "long_context_loss": 0.014584077361972545, + "mtp_acceptance_loss": 0.003889087296526012, + "output_kl": 0.04861359120657515, "peak_memory_cost": 0.0, "prefill_latency_cost": 0.0, - "task_loss_delta": 0.043663843738269315, - "token_disagreement": 0.036946329316997116 + "task_loss_delta": 0.03159883428427385, + "token_disagreement": 0.026737475163616336 }, "module_path": "model.language_model.layers.37.mlp.shared_expert_gate", "outlier_strategy": "none", "parameters": 2048, - "predicted_loss": 0.036677628740146226, - "reason": "selected by marginal quality gain per storage bit", - "role": "expert", + "predicted_loss": 0.02654302079879003, + "reason": "protected router policy", + "role": "router", "scale_strategy": "group-affine", "strategy_metadata": { - "selected_from_candidates": 7, - "storage_bpw": 7.0 + "selected_from_candidates": 3, + "storage_bpw": 9.0 }, "tensor": "model.language_model.layers.37.mlp.shared_expert_gate.weight" }, @@ -29222,31 +29222,31 @@ "tensor": "model.language_model.layers.38.mlp.shared_expert.up_proj.weight" }, { - "bits": 6, - "group_size": 32, - "method": "affine", + "bits": 16, + "group_size": null, + "method": "bf16", "metrics": { - "cosine_distance": 0.03022881489572491, + "cosine_distance": 0.0, "decode_latency_cost": 0.0, - "hidden_state_error": 0.05374011537017762, - "long_context_loss": 0.020152543263816602, - "mtp_acceptance_loss": 0.005374011537017762, - "output_kl": 0.06717514421272203, + "hidden_state_error": 0.0, + "long_context_loss": 0.0, + "mtp_acceptance_loss": 0.0, + "output_kl": 0.0, "peak_memory_cost": 0.0, "prefill_latency_cost": 0.0, - "task_loss_delta": 0.043663843738269315, - "token_disagreement": 0.036946329316997116 + "task_loss_delta": 0.0, + "token_disagreement": 0.0 }, "module_path": "model.language_model.layers.38.mlp.shared_expert_gate", "outlier_strategy": "none", "parameters": 2048, - "predicted_loss": 0.036677628740146226, - "reason": "selected by marginal quality gain per storage bit", - "role": "expert", - "scale_strategy": "group-affine", + "predicted_loss": 0.0, + "reason": "protected router policy", + "role": "router", + "scale_strategy": "none", "strategy_metadata": { - "selected_from_candidates": 7, - "storage_bpw": 7.0 + "selected_from_candidates": 3, + "storage_bpw": 16.0 }, "tensor": "model.language_model.layers.38.mlp.shared_expert_gate.weight" }, @@ -29541,31 +29541,31 @@ "tensor": "model.language_model.layers.39.mlp.shared_expert.up_proj.weight" }, { - "bits": 6, - "group_size": 32, - "method": "affine", + "bits": 16, + "group_size": null, + "method": "bf16", "metrics": { - "cosine_distance": 0.03022881489572491, + "cosine_distance": 0.0, "decode_latency_cost": 0.0, - "hidden_state_error": 0.05374011537017762, - "long_context_loss": 0.020152543263816602, - "mtp_acceptance_loss": 0.005374011537017762, - "output_kl": 0.06717514421272203, + "hidden_state_error": 0.0, + "long_context_loss": 0.0, + "mtp_acceptance_loss": 0.0, + "output_kl": 0.0, "peak_memory_cost": 0.0, "prefill_latency_cost": 0.0, - "task_loss_delta": 0.043663843738269315, - "token_disagreement": 0.036946329316997116 + "task_loss_delta": 0.0, + "token_disagreement": 0.0 }, "module_path": "model.language_model.layers.39.mlp.shared_expert_gate", "outlier_strategy": "none", "parameters": 2048, - "predicted_loss": 0.036677628740146226, - "reason": "selected by marginal quality gain per storage bit", - "role": "expert", - "scale_strategy": "group-affine", + "predicted_loss": 0.0, + "reason": "protected router policy", + "role": "router", + "scale_strategy": "none", "strategy_metadata": { - "selected_from_candidates": 7, - "storage_bpw": 7.0 + "selected_from_candidates": 3, + "storage_bpw": 16.0 }, "tensor": "model.language_model.layers.39.mlp.shared_expert_gate.weight" }, @@ -30343,8 +30343,8 @@ "minimum_mtp_speedup": 1.2, "minimum_quality_retention": 0.98 }, - "created_at": "2026-08-02T04:42:36.690943Z", - "effective_bpw": 5.999999988428959, + "created_at": "2026-08-05T13:38:03.627894Z", + "effective_bpw": 5.999999959946398, "evidence_kind": "architecture_prior", "global_validation_required": true, "group_size": 32, @@ -30368,6 +30368,7 @@ "affine", "awq", "dwq", + "gptq", "bf16" ] }, @@ -30390,7 +30391,7 @@ "parameters": 844640768 } }, - "nominal_bpw": 5.166136857126174, + "nominal_bpw": 5.166135034242241, "objective": { "cosine_distance": 0.05, "decode_latency_cost": 0.05, @@ -30410,7 +30411,7 @@ "schema_version": "axquant.plan.v1", "software_versions": { "ax_engine": null, - "axquant": "1.0.0", + "axquant": "1.2.0", "mlx": "0.32.0", "mlx_lm": "0.31.3", "pydantic": "2.13.4", @@ -30426,7 +30427,7 @@ }, "status": "planned", "target_bpw": 6.0, - "target_class": "4bit", + "target_class": "6bit", "target_mode": "low-memory", "warnings": [ "This report contains architecture priors, not calibration measurements.", @@ -30441,16 +30442,16 @@ "parameters": 24696061952 }, "6bit": { - "fraction": 0.2434611881590681, - "parameters": 8752873472 + "fraction": 0.2434625553220185, + "parameters": 8752922624 }, "8bit": { - "fraction": 0.019524909815543244, - "parameters": 701956096 + "fraction": 0.019523428722346983, + "parameters": 701902848 }, "bf16": { - "fraction": 0.05009290346215543, - "parameters": 1800931184 + "fraction": 0.0500930173924013, + "parameters": 1800935280 } } }