{
  "schema_version": 1,
  "role": "mac_quant_logit_metrics",
  "model": "/Users/oz/Documents/ChatGPT/qwen3.8-27b/qwen38-27b-quants/quantization-suite/artifacts/MLX-oQ5",
  "source_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0",
  "measured_at": "2026-08-16T02:07:13.532159+00:00",
  "inputs_sha256": "f1a0af6b9580739ebc9efa9375ae31aa6940dafe7ff6446097ed6cf8d9ab37de",
  "positions": 106,
  "mean_kl_divergence": 0.00603105514398161,
  "reference_perplexity": 9.84727010437735,
  "candidate_perplexity": 9.75069257534557,
  "perplexity_delta": -0.09657752903178007,
  "top1_token_agreement": 0.9811320754716981,
  "peak_unified_memory_gb": 19.17485066,
  "metal_memory_policy": {
    "device": {
      "device_name": "Apple M5 Pro",
      "max_recommended_working_set_size": 55662788608,
      "memory_size": 68719476736,
      "architecture": "applegpu_g17s",
      "max_buffer_length": 41747087360,
      "resource_limit": 499000
    },
    "cache_limit_bytes": 256000000,
    "wired_limit_bytes": 50096509747,
    "previous_cache_limit_bytes": 65283502899,
    "previous_wired_limit_bytes": 0,
    "warnings": []
  },
  "cases": [
    {
      "id": "release",
      "positions": 21,
      "mean_kl_divergence": 0.00826191995292902,
      "reference_perplexity": 105.96934764067565,
      "candidate_perplexity": 107.76979223376128,
      "top1_token_agreement": 1.0
    },
    {
      "id": "math",
      "positions": 32,
      "mean_kl_divergence": 0.004615321755409241,
      "reference_perplexity": 2.852022040669331,
      "candidate_perplexity": 2.7834795646020063,
      "top1_token_agreement": 1.0
    },
    {
      "id": "code",
      "positions": 30,
      "mean_kl_divergence": 0.0024357037618756294,
      "reference_perplexity": 2.682231762957344,
      "candidate_perplexity": 2.6615171884388524,
      "top1_token_agreement": 1.0
    },
    {
      "id": "multilingual",
      "positions": 23,
      "mean_kl_divergence": 0.010653483681380749,
      "reference_perplexity": 34.40712742702027,
      "candidate_perplexity": 33.83230252682042,
      "top1_token_agreement": 0.9130434782608695
    }
  ],
  "warnings": [
    "KL is measured on fixed original text, not a public benchmark.",
    "BF16 log-probabilities are stored in float16 after float32 log-softmax; reported KL therefore has finite-storage approximation error.",
    "The stock MLX-VLM logit scorer ignored 29 strict-loader extras, all proven to be under language_model.mtp; native oMLX validation separately loaded and tested MTP."
  ],
  "metric_ignored_mtp_tensors": [
    "language_model.mtp.fc.weight",
    "language_model.mtp.layers.0.input_layernorm.weight",
    "language_model.mtp.layers.0.mlp.down_proj.biases",
    "language_model.mtp.layers.0.mlp.down_proj.scales",
    "language_model.mtp.layers.0.mlp.down_proj.weight",
    "language_model.mtp.layers.0.mlp.gate_proj.biases",
    "language_model.mtp.layers.0.mlp.gate_proj.scales",
    "language_model.mtp.layers.0.mlp.gate_proj.weight",
    "language_model.mtp.layers.0.mlp.up_proj.biases",
    "language_model.mtp.layers.0.mlp.up_proj.scales",
    "language_model.mtp.layers.0.mlp.up_proj.weight",
    "language_model.mtp.layers.0.post_attention_layernorm.weight",
    "language_model.mtp.layers.0.self_attn.k_norm.weight",
    "language_model.mtp.layers.0.self_attn.k_proj.biases",
    "language_model.mtp.layers.0.self_attn.k_proj.scales",
    "language_model.mtp.layers.0.self_attn.k_proj.weight",
    "language_model.mtp.layers.0.self_attn.o_proj.biases",
    "language_model.mtp.layers.0.self_attn.o_proj.scales",
    "language_model.mtp.layers.0.self_attn.o_proj.weight",
    "language_model.mtp.layers.0.self_attn.q_norm.weight",
    "language_model.mtp.layers.0.self_attn.q_proj.biases",
    "language_model.mtp.layers.0.self_attn.q_proj.scales",
    "language_model.mtp.layers.0.self_attn.q_proj.weight",
    "language_model.mtp.layers.0.self_attn.v_proj.biases",
    "language_model.mtp.layers.0.self_attn.v_proj.scales",
    "language_model.mtp.layers.0.self_attn.v_proj.weight",
    "language_model.mtp.norm.weight",
    "language_model.mtp.pre_fc_norm_embedding.weight",
    "language_model.mtp.pre_fc_norm_hidden.weight."
  ],
  "dominance": {
    "candidate": {
      "functional_passed": true,
      "functional_failures": [],
      "text_passed": 10,
      "text_total": 10,
      "tools_passed": 5,
      "tools_total": 5,
      "vision_passed": 3,
      "vision_total": 3,
      "mtp_passed": true,
      "mtp_acceptance_rate": 0.96875,
      "mtp_speedup": 2.4515366430260044,
      "mean_semantic_similarity": 0.8852966785430908,
      "artifact_bytes": 20314066494,
      "peak_unified_memory_gb": 21.0919424,
      "average_generation_tps": 19.016428184965864,
      "mean_kl_divergence": 0.00603105514398161,
      "reference_perplexity": 9.84727010437735,
      "candidate_perplexity": 9.75069257534557,
      "perplexity_delta": -0.09657752903178007,
      "top1_token_agreement": 0.9811320754716981,
      "logit_positions": 106,
      "logit_inputs_sha256": "f1a0af6b9580739ebc9efa9375ae31aa6940dafe7ff6446097ed6cf8d9ab37de"
    },
    "baselines": {
      "MLX-5bit": {
        "functional_passed": true,
        "functional_failures": [],
        "text_passed": 10,
        "text_total": 10,
        "tools_passed": 5,
        "tools_total": 5,
        "vision_passed": 3,
        "vision_total": 3,
        "mtp_passed": true,
        "mtp_acceptance_rate": 0.9545454545454546,
        "mtp_speedup": 1.2124763256514137,
        "mean_semantic_similarity": 0.9520436644554138,
        "artifact_bytes": 20308322354,
        "peak_unified_memory_gb": 21.708525014,
        "average_generation_tps": 11.310333833788317,
        "mean_kl_divergence": 0.00583227794125395,
        "reference_perplexity": 9.84727010437735,
        "candidate_perplexity": 9.769640361449891,
        "perplexity_delta": -0.07762974292745817,
        "top1_token_agreement": 0.9716981132075472,
        "logit_positions": 106,
        "logit_inputs_sha256": "f1a0af6b9580739ebc9efa9375ae31aa6940dafe7ff6446097ed6cf8d9ab37de"
      },
      "MLX-6bit": {
        "functional_passed": true,
        "functional_failures": [],
        "text_passed": 10,
        "text_total": 10,
        "tools_passed": 5,
        "tools_total": 5,
        "vision_passed": 3,
        "vision_total": 3,
        "mtp_passed": true,
        "mtp_acceptance_rate": 0.9411764705882353,
        "mtp_speedup": 1.261565462823067,
        "mean_semantic_similarity": 0.988053274154663,
        "artifact_bytes": 23669991658,
        "peak_unified_memory_gb": 25.573573157,
        "average_generation_tps": 9.953450085041483,
        "mean_kl_divergence": 0.0015874907093228034,
        "reference_perplexity": 9.84727010437735,
        "candidate_perplexity": 9.821571581388875,
        "perplexity_delta": -0.025698522988474082,
        "top1_token_agreement": 1.0,
        "logit_positions": 106,
        "logit_inputs_sha256": "f1a0af6b9580739ebc9efa9375ae31aa6940dafe7ff6446097ed6cf8d9ab37de"
      },
      "MLX-oQ4e-MTP": {
        "functional_passed": true,
        "functional_failures": [],
        "text_passed": 10,
        "text_total": 10,
        "tools_passed": 5,
        "tools_total": 5,
        "vision_passed": 3,
        "vision_total": 3,
        "mtp_passed": true,
        "mtp_acceptance_rate": 0.968421052631579,
        "mtp_speedup": 2.3563535911602207,
        "mean_semantic_similarity": 0.8991085827350617,
        "artifact_bytes": 16994902573,
        "peak_unified_memory_gb": 17.785847808,
        "average_generation_tps": 21.81981842061714,
        "mean_kl_divergence": 0.016078564918266154,
        "reference_perplexity": 9.84727010437735,
        "candidate_perplexity": 10.072695109337184,
        "perplexity_delta": 0.22542500495983475,
        "top1_token_agreement": 0.9433962264150944,
        "logit_positions": 106,
        "logit_inputs_sha256": "f1a0af6b9580739ebc9efa9375ae31aa6940dafe7ff6446097ed6cf8d9ab37de"
      }
    }
  },
  "selection": "selected"
}