| { |
| "schema_version": 1, |
| "role": "mac_quant_logit_metrics", |
| "model": "/Users/oz/Documents/ChatGPT/qwen3.8-27b/qwen38-27b-quants/quantization-suite/artifacts/MLX-5bit-Group32", |
| "source_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", |
| "measured_at": "2026-08-16T01:30:05.600566+00:00", |
| "inputs_sha256": "f1a0af6b9580739ebc9efa9375ae31aa6940dafe7ff6446097ed6cf8d9ab37de", |
| "positions": 106, |
| "mean_kl_divergence": 0.004990289304054009, |
| "reference_perplexity": 9.84727010437735, |
| "candidate_perplexity": 9.883995913928073, |
| "perplexity_delta": 0.03672580955072391, |
| "top1_token_agreement": 0.9528301886792453, |
| "peak_unified_memory_gb": 20.314038144, |
| "metal_memory_policy": { |
| "device": { |
| "device_name": "Apple M5 Pro", |
| "max_recommended_working_set_size": 55662788608, |
| "memory_size": 68719476736, |
| "architecture": "applegpu_g17s", |
| "max_buffer_length": 41747087360, |
| "resource_limit": 499000 |
| }, |
| "cache_limit_bytes": 256000000, |
| "wired_limit_bytes": 50096509747, |
| "previous_cache_limit_bytes": 65283502899, |
| "previous_wired_limit_bytes": 0, |
| "warnings": [] |
| }, |
| "cases": [ |
| { |
| "id": "release", |
| "positions": 21, |
| "mean_kl_divergence": 0.005657650530338287, |
| "reference_perplexity": 105.96934764067565, |
| "candidate_perplexity": 108.92524694120041, |
| "top1_token_agreement": 1.0 |
| }, |
| { |
| "id": "math", |
| "positions": 32, |
| "mean_kl_divergence": 0.002653036266565323, |
| "reference_perplexity": 2.852022040669331, |
| "candidate_perplexity": 2.8364977771812123, |
| "top1_token_agreement": 1.0 |
| }, |
| { |
| "id": "code", |
| "positions": 30, |
| "mean_kl_divergence": 0.003575812326744199, |
| "reference_perplexity": 2.682231762957344, |
| "candidate_perplexity": 2.6788740507542905, |
| "top1_token_agreement": 0.9333333333333333 |
| }, |
| { |
| "id": "multilingual", |
| "positions": 23, |
| "mean_kl_divergence": 0.009477759711444378, |
| "reference_perplexity": 34.40712742702027, |
| "candidate_perplexity": 34.45066238752625, |
| "top1_token_agreement": 0.8695652173913043 |
| } |
| ], |
| "warnings": [ |
| "KL is measured on fixed original text, not a public benchmark.", |
| "BF16 log-probabilities are stored in float16 after float32 log-softmax; reported KL therefore has finite-storage approximation error." |
| ], |
| "dominance": { |
| "candidate": { |
| "functional_passed": true, |
| "functional_failures": [], |
| "text_passed": 10, |
| "text_total": 10, |
| "tools_passed": 5, |
| "tools_total": 5, |
| "vision_passed": 3, |
| "vision_total": 3, |
| "mtp_passed": true, |
| "mtp_acceptance_rate": 0.9545454545454546, |
| "mtp_speedup": 1.1150232750110602, |
| "mean_semantic_similarity": 0.9774516999721528, |
| "artifact_bytes": 21989153029, |
| "peak_unified_memory_gb": 23.639191525, |
| "average_generation_tps": 12.568759032272249, |
| "mean_kl_divergence": 0.004990289304054009, |
| "reference_perplexity": 9.84727010437735, |
| "candidate_perplexity": 9.883995913928073, |
| "perplexity_delta": 0.03672580955072391, |
| "top1_token_agreement": 0.9528301886792453, |
| "logit_positions": 106, |
| "logit_inputs_sha256": "f1a0af6b9580739ebc9efa9375ae31aa6940dafe7ff6446097ed6cf8d9ab37de" |
| }, |
| "baselines": { |
| "MLX-5bit": { |
| "functional_passed": true, |
| "functional_failures": [], |
| "text_passed": 10, |
| "text_total": 10, |
| "tools_passed": 5, |
| "tools_total": 5, |
| "vision_passed": 3, |
| "vision_total": 3, |
| "mtp_passed": true, |
| "mtp_acceptance_rate": 0.9545454545454546, |
| "mtp_speedup": 1.2124763256514137, |
| "mean_semantic_similarity": 0.9520436644554138, |
| "artifact_bytes": 20308322354, |
| "peak_unified_memory_gb": 21.708525014, |
| "average_generation_tps": 11.310333833788317, |
| "mean_kl_divergence": 0.00583227794125395, |
| "reference_perplexity": 9.84727010437735, |
| "candidate_perplexity": 9.769640361449891, |
| "perplexity_delta": -0.07762974292745817, |
| "top1_token_agreement": 0.9716981132075472, |
| "logit_positions": 106, |
| "logit_inputs_sha256": "f1a0af6b9580739ebc9efa9375ae31aa6940dafe7ff6446097ed6cf8d9ab37de" |
| }, |
| "MLX-6bit": { |
| "functional_passed": true, |
| "functional_failures": [], |
| "text_passed": 10, |
| "text_total": 10, |
| "tools_passed": 5, |
| "tools_total": 5, |
| "vision_passed": 3, |
| "vision_total": 3, |
| "mtp_passed": true, |
| "mtp_acceptance_rate": 0.9411764705882353, |
| "mtp_speedup": 1.261565462823067, |
| "mean_semantic_similarity": 0.988053274154663, |
| "artifact_bytes": 23669991658, |
| "peak_unified_memory_gb": 25.573573157, |
| "average_generation_tps": 9.953450085041483, |
| "mean_kl_divergence": 0.0015874907093228034, |
| "reference_perplexity": 9.84727010437735, |
| "candidate_perplexity": 9.821571581388875, |
| "perplexity_delta": -0.025698522988474082, |
| "top1_token_agreement": 1.0, |
| "logit_positions": 106, |
| "logit_inputs_sha256": "f1a0af6b9580739ebc9efa9375ae31aa6940dafe7ff6446097ed6cf8d9ab37de" |
| } |
| } |
| }, |
| "selection": "selected" |
| } |
|
|
