{
  "schema_version": 1,
  "role": "mac_quant_logit_metrics",
  "model": "/Users/oz/Documents/ChatGPT/qwen3.8-27b/qwen38-27b-quants/quantization-suite/artifacts/MLX-6bit-Group32",
  "source_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0",
  "measured_at": "2026-08-16T01:34:55.311732+00:00",
  "inputs_sha256": "f1a0af6b9580739ebc9efa9375ae31aa6940dafe7ff6446097ed6cf8d9ab37de",
  "positions": 106,
  "mean_kl_divergence": 0.001391285889834728,
  "reference_perplexity": 9.84727010437735,
  "candidate_perplexity": 9.812861698114856,
  "perplexity_delta": -0.034408406262492974,
  "top1_token_agreement": 0.9905660377358491,
  "peak_unified_memory_gb": 23.662099752,
  "metal_memory_policy": {
    "device": {
      "device_name": "Apple M5 Pro",
      "max_recommended_working_set_size": 55662788608,
      "memory_size": 68719476736,
      "architecture": "applegpu_g17s",
      "max_buffer_length": 41747087360,
      "resource_limit": 499000
    },
    "cache_limit_bytes": 256000000,
    "wired_limit_bytes": 50096509747,
    "previous_cache_limit_bytes": 65283502899,
    "previous_wired_limit_bytes": 0,
    "warnings": []
  },
  "cases": [
    {
      "id": "release",
      "positions": 21,
      "mean_kl_divergence": 0.0018035308457911015,
      "reference_perplexity": 105.96934764067565,
      "candidate_perplexity": 104.76209124887848,
      "top1_token_agreement": 1.0
    },
    {
      "id": "math",
      "positions": 32,
      "mean_kl_divergence": 0.0006423601298592985,
      "reference_perplexity": 2.852022040669331,
      "candidate_perplexity": 2.844368669232061,
      "top1_token_agreement": 0.96875
    },
    {
      "id": "code",
      "positions": 30,
      "mean_kl_divergence": 0.0007387612713500857,
      "reference_perplexity": 2.682231762957344,
      "candidate_perplexity": 2.6815580768171086,
      "top1_token_agreement": 1.0
    },
    {
      "id": "multilingual",
      "positions": 23,
      "mean_kl_divergence": 0.002907991176471114,
      "reference_perplexity": 34.40712742702027,
      "candidate_perplexity": 34.35197702879219,
      "top1_token_agreement": 1.0
    }
  ],
  "warnings": [
    "KL is measured on fixed original text, not a public benchmark.",
    "BF16 log-probabilities are stored in float16 after float32 log-softmax; reported KL therefore has finite-storage approximation error."
  ],
  "dominance": {
    "candidate": {
      "functional_passed": true,
      "functional_failures": [],
      "text_passed": 10,
      "text_total": 10,
      "tools_passed": 5,
      "tools_total": 5,
      "vision_passed": 3,
      "vision_total": 3,
      "mtp_passed": true,
      "mtp_acceptance_rate": 0.9411764705882353,
      "mtp_speedup": 1.2063376437161382,
      "mean_semantic_similarity": 0.9533392310142517,
      "artifact_bytes": 25350822273,
      "peak_unified_memory_gb": 27.489369028,
      "average_generation_tps": 11.107088812559562,
      "mean_kl_divergence": 0.001391285889834728,
      "reference_perplexity": 9.84727010437735,
      "candidate_perplexity": 9.812861698114856,
      "perplexity_delta": -0.034408406262492974,
      "top1_token_agreement": 0.9905660377358491,
      "logit_positions": 106,
      "logit_inputs_sha256": "f1a0af6b9580739ebc9efa9375ae31aa6940dafe7ff6446097ed6cf8d9ab37de"
    },
    "baselines": {
      "MLX-6bit": {
        "functional_passed": true,
        "functional_failures": [],
        "text_passed": 10,
        "text_total": 10,
        "tools_passed": 5,
        "tools_total": 5,
        "vision_passed": 3,
        "vision_total": 3,
        "mtp_passed": true,
        "mtp_acceptance_rate": 0.9411764705882353,
        "mtp_speedup": 1.261565462823067,
        "mean_semantic_similarity": 0.988053274154663,
        "artifact_bytes": 23669991658,
        "peak_unified_memory_gb": 25.573573157,
        "average_generation_tps": 9.953450085041483,
        "mean_kl_divergence": 0.0015874907093228034,
        "reference_perplexity": 9.84727010437735,
        "candidate_perplexity": 9.821571581388875,
        "perplexity_delta": -0.025698522988474082,
        "top1_token_agreement": 1.0,
        "logit_positions": 106,
        "logit_inputs_sha256": "f1a0af6b9580739ebc9efa9375ae31aa6940dafe7ff6446097ed6cf8d9ab37de"
      }
    }
  },
  "selection": "selected"
}