mac_metrics.json

{
  "schema_version": 1,
  "role": "mac_quant_logit_metrics",
  "model": "/Users/oz/Documents/ChatGPT/qwen3.8-27b/qwen38-27b-quants/quantization-suite/artifacts/MLX-8bit-Group32",
  "source_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0",
  "measured_at": "2026-08-16T01:40:54.669533+00:00",
  "inputs_sha256": "f1a0af6b9580739ebc9efa9375ae31aa6940dafe7ff6446097ed6cf8d9ab37de",
  "positions": 106,
  "mean_kl_divergence": 0.0003183703201840509,
  "reference_perplexity": 9.84727010437735,
  "candidate_perplexity": 9.853291632468812,
  "perplexity_delta": 0.0060215280914626135,
  "top1_token_agreement": 0.9811320754716981,
  "peak_unified_memory_gb": 30.370970545,
  "metal_memory_policy": {
    "device": {
      "device_name": "Apple M5 Pro",
      "max_recommended_working_set_size": 55662788608,
      "memory_size": 68719476736,
      "architecture": "applegpu_g17s",
      "max_buffer_length": 41747087360,
      "resource_limit": 499000
    },
    "cache_limit_bytes": 256000000,
    "wired_limit_bytes": 50096509747,
    "previous_cache_limit_bytes": 65283502899,
    "previous_wired_limit_bytes": 0,
    "warnings": []
  },
  "cases": [
    {
      "id": "release",
      "positions": 21,
      "mean_kl_divergence": 0.0002516350068617612,
      "reference_perplexity": 105.96934764067565,
      "candidate_perplexity": 106.91228116633563,
      "top1_token_agreement": 1.0
    },
    {
      "id": "math",
      "positions": 32,
      "mean_kl_divergence": 0.00027180201141163707,
      "reference_perplexity": 2.852022040669331,
      "candidate_perplexity": 2.839981371955365,
      "top1_token_agreement": 1.0
    },
    {
      "id": "code",
      "positions": 30,
      "mean_kl_divergence": 0.00015606466331519186,
      "reference_perplexity": 2.682231762957344,
      "candidate_perplexity": 2.6692335442419357,
      "top1_token_agreement": 0.9666666666666667
    },
    {
      "id": "multilingual",
      "positions": 23,
      "mean_kl_divergence": 0.000655796320643276,
      "reference_perplexity": 34.40712742702027,
      "candidate_perplexity": 34.64713745598045,
      "top1_token_agreement": 0.9565217391304348
    }
  ],
  "warnings": [
    "KL is measured on fixed original text, not a public benchmark.",
    "BF16 log-probabilities are stored in float16 after float32 log-softmax; reported KL therefore has finite-storage approximation error."
  ],
  "dominance": {
    "candidate": {
      "functional_passed": true,
      "functional_failures": [],
      "text_passed": 10,
      "text_total": 10,
      "tools_passed": 5,
      "tools_total": 5,
      "vision_passed": 3,
      "vision_total": 3,
      "mtp_passed": true,
      "mtp_acceptance_rate": 0.9545454545454546,
      "mtp_speedup": 1.4002487484524309,
      "mean_semantic_similarity": 0.9863808214664459,
      "artifact_bytes": 32074160991,
      "peak_unified_memory_gb": 35.199014081,
      "average_generation_tps": 8.52152581187538,
      "mean_kl_divergence": 0.0003183703201840509,
      "reference_perplexity": 9.84727010437735,
      "candidate_perplexity": 9.853291632468812,
      "perplexity_delta": 0.0060215280914626135,
      "top1_token_agreement": 0.9811320754716981,
      "logit_positions": 106,
      "logit_inputs_sha256": "f1a0af6b9580739ebc9efa9375ae31aa6940dafe7ff6446097ed6cf8d9ab37de"
    },
    "baselines": {
      "MLX-8bit": {
        "functional_passed": true,
        "functional_failures": [],
        "text_passed": 10,
        "text_total": 10,
        "tools_passed": 5,
        "tools_total": 5,
        "vision_passed": 3,
        "vision_total": 3,
        "mtp_passed": true,
        "mtp_acceptance_rate": 1.0,
        "mtp_speedup": 1.5430882573309594,
        "mean_semantic_similarity": 0.981384402513504,
        "artifact_bytes": 30393325415,
        "peak_unified_memory_gb": 33.264178369,
        "average_generation_tps": 7.443406726106941,
        "mean_kl_divergence": 0.00038159784052591277,
        "reference_perplexity": 9.84727010437735,
        "candidate_perplexity": 9.850808167988083,
        "perplexity_delta": 0.0035380636107333885,
        "top1_token_agreement": 0.9905660377358491,
        "logit_positions": 106,
        "logit_inputs_sha256": "f1a0af6b9580739ebc9efa9375ae31aa6940dafe7ff6446097ed6cf8d9ab37de"
      },
      "MLX-MXFP8": {
        "functional_passed": true,
        "functional_failures": [],
        "text_passed": 10,
        "text_total": 10,
        "tools_passed": 5,
        "tools_total": 5,
        "vision_passed": 3,
        "vision_total": 3,
        "mtp_passed": true,
        "mtp_acceptance_rate": 1.0,
        "mtp_speedup": 1.2421434584414472,
        "mean_semantic_similarity": 0.9667035639286041,
        "artifact_bytes": 29552791900,
        "peak_unified_memory_gb": 28.331811902,
        "average_generation_tps": 8.792077918162095,
        "mean_kl_divergence": 0.025145749157329776,
        "reference_perplexity": 9.84727010437735,
        "candidate_perplexity": 9.912899427959013,
        "perplexity_delta": 0.06562932358166407,
        "top1_token_agreement": 0.9245283018867925,
        "logit_positions": 106,
        "logit_inputs_sha256": "f1a0af6b9580739ebc9efa9375ae31aa6940dafe7ff6446097ed6cf8d9ab37de"
      }
    }
  },
  "selection": "selected"
}