{ "schema_version": 1, "role": "mac_quant_logit_metrics", "model": "/Users/oz/Documents/ChatGPT/qwen3.8-27b/qwen38-27b-quants/quantization-suite/artifacts/MLX-oQ8", "source_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "measured_at": "2026-08-16T02:05:59.277180+00:00", "inputs_sha256": "f1a0af6b9580739ebc9efa9375ae31aa6940dafe7ff6446097ed6cf8d9ab37de", "positions": 106, "mean_kl_divergence": 0.0007939068249092912, "reference_perplexity": 9.84727010437735, "candidate_perplexity": 9.89084181958065, "perplexity_delta": 0.043571715203301054, "top1_token_agreement": 1.0, "peak_unified_memory_gb": 28.69873579, "metal_memory_policy": { "device": { "device_name": "Apple M5 Pro", "max_recommended_working_set_size": 55662788608, "memory_size": 68719476736, "architecture": "applegpu_g17s", "max_buffer_length": 41747087360, "resource_limit": 499000 }, "cache_limit_bytes": 256000000, "wired_limit_bytes": 50096509747, "previous_cache_limit_bytes": 65283502899, "previous_wired_limit_bytes": 0, "warnings": [] }, "cases": [ { "id": "release", "positions": 21, "mean_kl_divergence": 0.0008054509526118636, "reference_perplexity": 105.96934764067565, "candidate_perplexity": 106.5703617028238, "top1_token_agreement": 1.0 }, { "id": "math", "positions": 32, "mean_kl_divergence": 0.0007041142089292407, "reference_perplexity": 2.852022040669331, "candidate_perplexity": 2.8345298272312576, "top1_token_agreement": 1.0 }, { "id": "code", "positions": 30, "mean_kl_divergence": 0.0003090364334639162, "reference_perplexity": 2.682231762957344, "candidate_perplexity": 2.6772788144620736, "top1_token_agreement": 1.0 }, { "id": "multilingual", "positions": 23, "mean_kl_divergence": 0.0015407350147143006, "reference_perplexity": 34.40712742702027, "candidate_perplexity": 35.318878819504626, "top1_token_agreement": 1.0 } ], "warnings": [ "KL is measured on fixed original text, not a public benchmark.", "BF16 log-probabilities are stored in float16 after float32 log-softmax; reported KL therefore has finite-storage approximation error.", "The stock MLX-VLM logit scorer ignored 29 strict-loader extras, all proven to be under language_model.mtp; native oMLX validation separately loaded and tested MTP." ], "metric_ignored_mtp_tensors": [ "language_model.mtp.fc.weight", "language_model.mtp.layers.0.input_layernorm.weight", "language_model.mtp.layers.0.mlp.down_proj.biases", "language_model.mtp.layers.0.mlp.down_proj.scales", "language_model.mtp.layers.0.mlp.down_proj.weight", "language_model.mtp.layers.0.mlp.gate_proj.biases", "language_model.mtp.layers.0.mlp.gate_proj.scales", "language_model.mtp.layers.0.mlp.gate_proj.weight", "language_model.mtp.layers.0.mlp.up_proj.biases", "language_model.mtp.layers.0.mlp.up_proj.scales", "language_model.mtp.layers.0.mlp.up_proj.weight", "language_model.mtp.layers.0.post_attention_layernorm.weight", "language_model.mtp.layers.0.self_attn.k_norm.weight", "language_model.mtp.layers.0.self_attn.k_proj.biases", "language_model.mtp.layers.0.self_attn.k_proj.scales", "language_model.mtp.layers.0.self_attn.k_proj.weight", "language_model.mtp.layers.0.self_attn.o_proj.biases", "language_model.mtp.layers.0.self_attn.o_proj.scales", "language_model.mtp.layers.0.self_attn.o_proj.weight", "language_model.mtp.layers.0.self_attn.q_norm.weight", "language_model.mtp.layers.0.self_attn.q_proj.biases", "language_model.mtp.layers.0.self_attn.q_proj.scales", "language_model.mtp.layers.0.self_attn.q_proj.weight", "language_model.mtp.layers.0.self_attn.v_proj.biases", "language_model.mtp.layers.0.self_attn.v_proj.scales", "language_model.mtp.layers.0.self_attn.v_proj.weight", "language_model.mtp.norm.weight", "language_model.mtp.pre_fc_norm_embedding.weight", "language_model.mtp.pre_fc_norm_hidden.weight." ], "dominance": { "candidate": { "functional_passed": true, "functional_failures": [], "text_passed": 10, "text_total": 10, "tools_passed": 5, "tools_total": 5, "vision_passed": 3, "vision_total": 3, "mtp_passed": true, "mtp_acceptance_rate": 1.0, "mtp_speedup": 2.4211956521739126, "mean_semantic_similarity": 0.8972687065601349, "artifact_bytes": 30024788604, "peak_unified_memory_gb": 28.69873579, "average_generation_tps": 13.513484058818289, "mean_kl_divergence": 0.0007939068249092912, "reference_perplexity": 9.84727010437735, "candidate_perplexity": 9.89084181958065, "perplexity_delta": 0.043571715203301054, "top1_token_agreement": 1.0, "logit_positions": 106, "logit_inputs_sha256": "f1a0af6b9580739ebc9efa9375ae31aa6940dafe7ff6446097ed6cf8d9ab37de" }, "baselines": { "MLX-8bit": { "functional_passed": true, "functional_failures": [], "text_passed": 10, "text_total": 10, "tools_passed": 5, "tools_total": 5, "vision_passed": 3, "vision_total": 3, "mtp_passed": true, "mtp_acceptance_rate": 1.0, "mtp_speedup": 1.5430882573309594, "mean_semantic_similarity": 0.981384402513504, "artifact_bytes": 30393325415, "peak_unified_memory_gb": 33.264178369, "average_generation_tps": 7.443406726106941, "mean_kl_divergence": 0.00038159784052591277, "reference_perplexity": 9.84727010437735, "candidate_perplexity": 9.850808167988083, "perplexity_delta": 0.0035380636107333885, "top1_token_agreement": 0.9905660377358491, "logit_positions": 106, "logit_inputs_sha256": "f1a0af6b9580739ebc9efa9375ae31aa6940dafe7ff6446097ed6cf8d9ab37de" }, "MLX-MXFP8": { "functional_passed": true, "functional_failures": [], "text_passed": 10, "text_total": 10, "tools_passed": 5, "tools_total": 5, "vision_passed": 3, "vision_total": 3, "mtp_passed": true, "mtp_acceptance_rate": 1.0, "mtp_speedup": 1.2421434584414472, "mean_semantic_similarity": 0.9667035639286041, "artifact_bytes": 29552791900, "peak_unified_memory_gb": 28.331811902, "average_generation_tps": 8.792077918162095, "mean_kl_divergence": 0.025145749157329776, "reference_perplexity": 9.84727010437735, "candidate_perplexity": 9.912899427959013, "perplexity_delta": 0.06562932358166407, "top1_token_agreement": 0.9245283018867925, "logit_positions": 106, "logit_inputs_sha256": "f1a0af6b9580739ebc9efa9375ae31aa6940dafe7ff6446097ed6cf8d9ab37de" } } }, "selection": "selected" }