{
  "schema_version": 1,
  "role": "mac_quant_logit_metrics",
  "model": "/Users/oz/Documents/ChatGPT/qwen3.8-27b/qwen38-27b-quants/quantization-suite/artifacts/MLX-5bit-Group32",
  "source_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0",
  "measured_at": "2026-08-16T01:30:05.600566+00:00",
  "inputs_sha256": "f1a0af6b9580739ebc9efa9375ae31aa6940dafe7ff6446097ed6cf8d9ab37de",
  "positions": 106,
  "mean_kl_divergence": 0.004990289304054009,
  "reference_perplexity": 9.84727010437735,
  "candidate_perplexity": 9.883995913928073,
  "perplexity_delta": 0.03672580955072391,
  "top1_token_agreement": 0.9528301886792453,
  "peak_unified_memory_gb": 20.314038144,
  "metal_memory_policy": {
    "device": {
      "device_name": "Apple M5 Pro",
      "max_recommended_working_set_size": 55662788608,
      "memory_size": 68719476736,
      "architecture": "applegpu_g17s",
      "max_buffer_length": 41747087360,
      "resource_limit": 499000
    },
    "cache_limit_bytes": 256000000,
    "wired_limit_bytes": 50096509747,
    "previous_cache_limit_bytes": 65283502899,
    "previous_wired_limit_bytes": 0,
    "warnings": []
  },
  "cases": [
    {
      "id": "release",
      "positions": 21,
      "mean_kl_divergence": 0.005657650530338287,
      "reference_perplexity": 105.96934764067565,
      "candidate_perplexity": 108.92524694120041,
      "top1_token_agreement": 1.0
    },
    {
      "id": "math",
      "positions": 32,
      "mean_kl_divergence": 0.002653036266565323,
      "reference_perplexity": 2.852022040669331,
      "candidate_perplexity": 2.8364977771812123,
      "top1_token_agreement": 1.0
    },
    {
      "id": "code",
      "positions": 30,
      "mean_kl_divergence": 0.003575812326744199,
      "reference_perplexity": 2.682231762957344,
      "candidate_perplexity": 2.6788740507542905,
      "top1_token_agreement": 0.9333333333333333
    },
    {
      "id": "multilingual",
      "positions": 23,
      "mean_kl_divergence": 0.009477759711444378,
      "reference_perplexity": 34.40712742702027,
      "candidate_perplexity": 34.45066238752625,
      "top1_token_agreement": 0.8695652173913043
    }
  ],
  "warnings": [
    "KL is measured on fixed original text, not a public benchmark.",
    "BF16 log-probabilities are stored in float16 after float32 log-softmax; reported KL therefore has finite-storage approximation error."
  ],
  "dominance": {
    "candidate": {
      "functional_passed": true,
      "functional_failures": [],
      "text_passed": 10,
      "text_total": 10,
      "tools_passed": 5,
      "tools_total": 5,
      "vision_passed": 3,
      "vision_total": 3,
      "mtp_passed": true,
      "mtp_acceptance_rate": 0.9545454545454546,
      "mtp_speedup": 1.1150232750110602,
      "mean_semantic_similarity": 0.9774516999721528,
      "artifact_bytes": 21989153029,
      "peak_unified_memory_gb": 23.639191525,
      "average_generation_tps": 12.568759032272249,
      "mean_kl_divergence": 0.004990289304054009,
      "reference_perplexity": 9.84727010437735,
      "candidate_perplexity": 9.883995913928073,
      "perplexity_delta": 0.03672580955072391,
      "top1_token_agreement": 0.9528301886792453,
      "logit_positions": 106,
      "logit_inputs_sha256": "f1a0af6b9580739ebc9efa9375ae31aa6940dafe7ff6446097ed6cf8d9ab37de"
    },
    "baselines": {
      "MLX-5bit": {
        "functional_passed": true,
        "functional_failures": [],
        "text_passed": 10,
        "text_total": 10,
        "tools_passed": 5,
        "tools_total": 5,
        "vision_passed": 3,
        "vision_total": 3,
        "mtp_passed": true,
        "mtp_acceptance_rate": 0.9545454545454546,
        "mtp_speedup": 1.2124763256514137,
        "mean_semantic_similarity": 0.9520436644554138,
        "artifact_bytes": 20308322354,
        "peak_unified_memory_gb": 21.708525014,
        "average_generation_tps": 11.310333833788317,
        "mean_kl_divergence": 0.00583227794125395,
        "reference_perplexity": 9.84727010437735,
        "candidate_perplexity": 9.769640361449891,
        "perplexity_delta": -0.07762974292745817,
        "top1_token_agreement": 0.9716981132075472,
        "logit_positions": 106,
        "logit_inputs_sha256": "f1a0af6b9580739ebc9efa9375ae31aa6940dafe7ff6446097ed6cf8d9ab37de"
      },
      "MLX-6bit": {
        "functional_passed": true,
        "functional_failures": [],
        "text_passed": 10,
        "text_total": 10,
        "tools_passed": 5,
        "tools_total": 5,
        "vision_passed": 3,
        "vision_total": 3,
        "mtp_passed": true,
        "mtp_acceptance_rate": 0.9411764705882353,
        "mtp_speedup": 1.261565462823067,
        "mean_semantic_similarity": 0.988053274154663,
        "artifact_bytes": 23669991658,
        "peak_unified_memory_gb": 25.573573157,
        "average_generation_tps": 9.953450085041483,
        "mean_kl_divergence": 0.0015874907093228034,
        "reference_perplexity": 9.84727010437735,
        "candidate_perplexity": 9.821571581388875,
        "perplexity_delta": -0.025698522988474082,
        "top1_token_agreement": 1.0,
        "logit_positions": 106,
        "logit_inputs_sha256": "f1a0af6b9580739ebc9efa9375ae31aa6940dafe7ff6446097ed6cf8d9ab37de"
      }
    }
  },
  "selection": "selected"
}