{
  "schema_version": 1,
  "role": "mac_quant_logit_metrics",
  "model": "/Users/oz/Documents/ChatGPT/qwen3.8-27b/qwen38-27b-quants/quantization-suite/artifacts/MLX-5bit-Group128",
  "source_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0",
  "measured_at": "2026-08-16T01:51:49.184662+00:00",
  "inputs_sha256": "f1a0af6b9580739ebc9efa9375ae31aa6940dafe7ff6446097ed6cf8d9ab37de",
  "positions": 106,
  "mean_kl_divergence": 0.007879387376443395,
  "reference_perplexity": 9.84727010437735,
  "candidate_perplexity": 9.764229178940486,
  "perplexity_delta": -0.08304092543686359,
  "top1_token_agreement": 0.9905660377358491,
  "peak_unified_memory_gb": 17.804167926,
  "metal_memory_policy": {
    "device": {
      "device_name": "Apple M5 Pro",
      "max_recommended_working_set_size": 55662788608,
      "memory_size": 68719476736,
      "architecture": "applegpu_g17s",
      "max_buffer_length": 41747087360,
      "resource_limit": 499000
    },
    "cache_limit_bytes": 256000000,
    "wired_limit_bytes": 50096509747,
    "previous_cache_limit_bytes": 65283502899,
    "previous_wired_limit_bytes": 0,
    "warnings": []
  },
  "cases": [
    {
      "id": "release",
      "positions": 21,
      "mean_kl_divergence": 0.009627277962863445,
      "reference_perplexity": 105.96934764067565,
      "candidate_perplexity": 104.98872011380237,
      "top1_token_agreement": 1.0
    },
    {
      "id": "math",
      "positions": 32,
      "mean_kl_divergence": 0.005563782528042793,
      "reference_perplexity": 2.852022040669331,
      "candidate_perplexity": 2.8400412963201846,
      "top1_token_agreement": 1.0
    },
    {
      "id": "code",
      "positions": 30,
      "mean_kl_divergence": 0.002371579874306917,
      "reference_perplexity": 2.682231762957344,
      "candidate_perplexity": 2.647959665303878,
      "top1_token_agreement": 1.0
    },
    {
      "id": "multilingual",
      "positions": 23,
      "mean_kl_divergence": 0.016689294949173927,
      "reference_perplexity": 34.40712742702027,
      "candidate_perplexity": 34.13602848310655,
      "top1_token_agreement": 0.9565217391304348
    }
  ],
  "warnings": [
    "KL is measured on fixed original text, not a public benchmark.",
    "BF16 log-probabilities are stored in float16 after float32 log-softmax; reported KL therefore has finite-storage approximation error."
  ],
  "dominance": {
    "candidate": {
      "functional_passed": true,
      "functional_failures": [],
      "text_passed": 10,
      "text_total": 10,
      "tools_passed": 5,
      "tools_total": 5,
      "vision_passed": 3,
      "vision_total": 3,
      "mtp_passed": true,
      "mtp_acceptance_rate": 0.9545454545454546,
      "mtp_speedup": 1.04670949751592,
      "mean_semantic_similarity": 0.9368221938610077,
      "artifact_bytes": 19467900026,
      "peak_unified_memory_gb": 20.756765927,
      "average_generation_tps": 14.54951526929673,
      "mean_kl_divergence": 0.007879387376443395,
      "reference_perplexity": 9.84727010437735,
      "candidate_perplexity": 9.764229178940486,
      "perplexity_delta": -0.08304092543686359,
      "top1_token_agreement": 0.9905660377358491,
      "logit_positions": 106,
      "logit_inputs_sha256": "f1a0af6b9580739ebc9efa9375ae31aa6940dafe7ff6446097ed6cf8d9ab37de"
    },
    "baselines": {
      "MLX-5bit": {
        "functional_passed": true,
        "functional_failures": [],
        "text_passed": 10,
        "text_total": 10,
        "tools_passed": 5,
        "tools_total": 5,
        "vision_passed": 3,
        "vision_total": 3,
        "mtp_passed": true,
        "mtp_acceptance_rate": 0.9545454545454546,
        "mtp_speedup": 1.2124763256514137,
        "mean_semantic_similarity": 0.9520436644554138,
        "artifact_bytes": 20308322354,
        "peak_unified_memory_gb": 21.708525014,
        "average_generation_tps": 11.310333833788317,
        "mean_kl_divergence": 0.00583227794125395,
        "reference_perplexity": 9.84727010437735,
        "candidate_perplexity": 9.769640361449891,
        "perplexity_delta": -0.07762974292745817,
        "top1_token_agreement": 0.9716981132075472,
        "logit_positions": 106,
        "logit_inputs_sha256": "f1a0af6b9580739ebc9efa9375ae31aa6940dafe7ff6446097ed6cf8d9ab37de"
      }
    }
  },
  "selection": "selected"
}