card + metrics + charts + imatrix + Modelfiles

.gitattributes

@@ -33,3 +33,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+amalia-9b-pt.imatrix filter=lfs diff=lfs merge=lfs -text
+banner.png filter=lfs diff=lfs merge=lfs -text

CONTEXT.md

@@ -0,0 +1,14 @@
+# Context size
+
+AMALIA-9B is trained with a native context window of **32768 tokens** (`rope_theta 1e6`).
+
+Included Modelfiles set `num_ctx`:
+- `Modelfile.8k`  — 8192 (lightest RAM; good for chat)
+- `Modelfile.32k` — 32768 (native maximum)
+
+```bash
+ollama create amalia-9b-32k -f Modelfile.32k
+ollama run amalia-9b-32k
+```
+
+Larger contexts than 32K require RoPE scaling and will degrade quality — 32K is the honest native limit. KV-cache grows with context, so at 32K budget extra RAM on top of the weights.

Modelfile.32k

@@ -0,0 +1,4 @@
+FROM ./amalia-9b-Q4_K_M.gguf
+PARAMETER num_ctx 32768
+PARAMETER temperature 0.7
+PARAMETER top_p 0.9

Modelfile.8k

@@ -0,0 +1,4 @@
+FROM ./amalia-9b-Q4_K_M.gguf
+PARAMETER num_ctx 8192
+PARAMETER temperature 0.7
+PARAMETER top_p 0.9

README.md

@@ -0,0 +1,148 @@
+---
+license: apache-2.0
+base_model:
+  - amalia-llm/AMALIA-9B-0626-SFT
+base_model_relation: quantized
+library_name: llama.cpp
+pipeline_tag: text-generation
+tags:
+  - gguf
+  - quantized
+  - llama-cpp
+  - imatrix
+  - llama
+  - portuguese
+  - european-portuguese
+  - kld-measured
+  - conversational
+  - endpoints_compatible
+language:
+  - pt
+---
+
+![banner](banner.png)
+
+<div style="border:2px solid currentColor; font-family:ui-monospace,'SF Mono','Cascadia Mono',Consolas,monospace;">
+  <div style="border-bottom:1px solid currentColor; padding:6px 12px; font-size:11px; letter-spacing:3px; text-transform:uppercase; opacity:0.7; text-align:center;">KIKOCIS // EUROPEAN-PORTUGUESE LLM // IMATRIX GGUF + KLD</div>
+  <div style="padding:14px; display:flex; flex-wrap:wrap; align-items:center; justify-content:center; gap:18px;">
+<pre style="margin:0; font-size:9px; line-height:1.15;">
+   ╔═══════════════╗
+   ║  A M A L I A  ║   pt-PT
+   ╚═══════════════╝
+   ┌──┐ ┌──┐ ┌──┐ ┌──┐
+   │Q3│ │Q4│ │Q5│ │Q8│  ← ladder
+   └──┘ └──┘ └──┘ └──┘
+    ●────●────●────●
+      KLD vs F16
+</pre>
+    <div style="text-align:center;">
+      <div style="font-size:23px; font-weight:800; letter-spacing:1px;">AMALIA-9B · GGUF</div>
+      <div style="font-size:12.5px; letter-spacing:1px; opacity:0.8; margin-top:5px;"><span style="white-space:nowrap;">llama · 9B</span> · <span style="white-space:nowrap;">32K ctx</span> · <span style="white-space:nowrap;">imatrix (pt)</span> · <span style="white-space:nowrap;">KLD-measured</span></div>
+    </div>
+  </div>
+  <table style="display:table; table-layout:fixed; width:100%; margin:0; border-collapse:collapse; font-size:12px;">
+    <tr>
+      <td style="border-top:1px solid currentColor; border-right:1px solid currentColor; padding:8px 12px;"><div style="font-size:10px; letter-spacing:1px; opacity:0.6;">FORMAT</div><div style="font-weight:700;">GGUF (imatrix)</div></td>
+      <td style="border-top:1px solid currentColor; border-right:1px solid currentColor; padding:8px 12px;"><div style="font-size:10px; letter-spacing:1px; opacity:0.6;">SIZES</div><div style="font-weight:700;">3.1 – 9.1 GB</div></td>
+      <td style="border-top:1px solid currentColor; border-right:1px solid currentColor; padding:8px 12px;"><div style="font-size:10px; letter-spacing:1px; opacity:0.6;">ARCH</div><div style="font-weight:700;">Llama · 42L</div></td>
+      <td style="border-top:1px solid currentColor; padding:8px 12px;"><div style="font-size:10px; letter-spacing:1px; opacity:0.6;">CONTEXT</div><div style="font-weight:700;">32768</div></td>
+    </tr>
+    <tr>
+      <td style="border-top:1px solid currentColor; border-right:1px solid currentColor; padding:8px 12px;"><div style="font-size:10px; letter-spacing:1px; opacity:0.6;">IMATRIX</div><div style="font-weight:700;">pt-PT corpus</div></td>
+      <td style="border-top:1px solid currentColor; border-right:1px solid currentColor; padding:8px 12px;"><div style="font-size:10px; letter-spacing:1px; opacity:0.6;">VALIDATION</div><div style="font-weight:700;">KLD vs F16</div></td>
+      <td style="border-top:1px solid currentColor; border-right:1px solid currentColor; padding:8px 12px;"><div style="font-size:10px; letter-spacing:1px; opacity:0.6;">LANGUAGE</div><div style="font-weight:700;">Português (EU)</div></td>
+      <td style="border-top:1px solid currentColor; padding:8px 12px;"><div style="font-size:10px; letter-spacing:1px; opacity:0.6;">LICENSE</div><div style="font-weight:700;">Apache-2.0</div></td>
+    </tr>
+  </table>
+</div>
+
+# AMALIA-9B-0626-SFT — GGUF (imatrix + KLD)
+> imatrix-quantized GGUFs of **AMALIA-9B**, an open **European-Portuguese** instruction model — one of the first open LLMs developed in **Portugal**. Runs from **~3.1 GB** (IQ2_M) to **~9.1 GB** (Q8_0). Calibrated (imatrix) **on Portuguese text**, and every quant is measured against the full-precision model with **KLD** so you can pick a size with the fidelity cost in front of you. Credit: this is **the AMALIA-LLM project's** model — [`amalia-llm/AMALIA-9B-0626-SFT`](https://huggingface.co/amalia-llm/AMALIA-9B-0626-SFT); ours is the quant ladder + metrics.
+
+## ✅ Recommended files
+| Use case | File | Notes |
+|---|---|---|
+| ⭐ Best value | `amalia-9b-Q5_K_M.gguf` | The pick — **lower KLD than Q6_K** at 1 GB less (0.051 vs 0.078). ~6.1 GB. |
+| Safe default | `amalia-9b-Q4_K_M.gguf` | Standard K-quant, ~5.2 GB; noticeably less faithful than Q5 (KLD 0.22 vs 0.05). |
+| Max fidelity | `amalia-9b-Q8_0.gguf` | Lowest KLD (0.006) — near-lossless. ~9.1 GB. |
+| Tightest RAM | `amalia-9b-Q3_K_M.gguf` | ~4.2 GB — runs on 8 GB machines, still coherent. |
+| Smallest (aggressive) | `amalia-9b-IQ2_M.gguf` | ~3.1 GB — fits very tight RAM, at a real quality cost (top-1 61%). |
+
+## 📦 Files (the full ladder)
+| Quant | Bits | File size | Notes |
+|---|---|---:|---|
+| IQ2_M | ~2.7 | ~3.1 GB | Smallest — aggressive i-quant, real quality trade-off. |
+| Q3_K_M | 3 | ~4.2 GB | Smallest *comfortable* quant. |
+| IQ4_XS | ~4.3 | ~4.7 GB | Compact i-quant (imatrix). |
+| Q4_K_M | 4 | ~5.2 GB | Standard K-quant. |
+| Q5_K_M | 5 | ~6.1 GB | ⭐ Best quality/size. |
+| Q6_K | 6 | ~7.0 GB | Near-lossless. |
+| Q8_0 | 8 | ~9.1 GB | Highest precision. |
+<!-- K-quants (Q*_K) = solid general quants. I-quants (IQ*) = smaller, need an imatrix (shipped). Higher number = more bits = closer to the original + bigger. -->
+
+## 📊 Metrics — objective quality vs the F16 reference
+**KLD** (Kullback–Leibler divergence, nats) measures how far each quant's output distribution drifts from the full-precision model — the gold-standard fidelity metric (lower = closer). **Top-1 match** = how often the quant's top token agrees with F16. Measured with `llama-perplexity --kl-divergence` over a **Portuguese** corpus at **ctx 2048**, vs the F16 GGUF.
+
+| Model | Size GB | PPL | ΔPPL vs F16 | KLD mean | KLD p95 | Top-1 match |
+|---|---:|---:|---:|---:|---:|---:|
+| **F16 (reference)** | 17.1 | 20.04 | 0.000 | 0.0000 | 0.0000 | 100.0% |
+| Q8_0 | 9.06 | 20.20 | +0.177 | 0.0057 | 0.0198 | 96.38% |
+| Q6_K | 7.00 | 19.93 | -0.091 | 0.0779 | 0.1740 | 89.06% |
+| Q5_K_M | 6.08 | 19.80 | -0.227 | 0.0511 | 0.1747 | 88.89% |
+| Q4_K_M | 5.20 | 20.23 | +0.203 | 0.2230 | 0.7429 | 81.39% |
+| IQ4_XS | 4.70 | 20.78 | +0.757 | 0.2173 | 0.7600 | 80.55% |
+| Q3_K_M | 4.25 | 22.24 | +2.215 | 0.3258 | 1.1486 | 73.58% |
+| IQ2_M | 3.11 | 21.99 | +1.968 | 0.8211 | 2.9052 | 60.69% |
+
+<sub>Full per-quant reports in [`reports/`](reports); machine-readable summary in [`metrics/quant-summary.csv`](metrics/quant-summary.csv); SHA-256 of every file in [`reports/artifact-sha256sums.txt`](reports/artifact-sha256sums.txt). Note: this is an **instruction-tuned** model measured on **raw** text, so the absolute PPL is higher than a base model's — what matters here is the **relative** KLD/top-1 across quants.</sub>
+
+> **Why does Q5_K_M beat Q6_K on KLD here?** It's real and reproducible for this model: Q5_K_M's imatrix-weighted bit allocation is very effective, so it lands closer to F16 on mean-KLD than Q6_K (which stays marginally ahead on top-1). Both are near-lossless — so **Q5_K_M is the better value**, and Q8_0 is the true fidelity ceiling.
+
+> **A note on i-quants:** we tested the i-quants too. For *this* model the K-quants win at low bit — e.g. IQ3_M came out *worse* than Q3_K_M on KLD, so it isn't shipped. **IQ2_M is included** as the smallest option for very tight RAM, but with an honest quality cost (top-1 61%); if you can spare the space, Q3_K_M is a big step up.
+
+### 📈 Charts
+![kld vs size](metrics/chart-kld-vs-size.png)
+![top-1 match](metrics/chart-top1.png)
+
+## 🧮 Will it fit? (RAM/VRAM cheat-sheet)
+Memory ≈ weights + KV-cache (KV grows with context). Rough guide:
+| you have | comfortable quant | context |
+|---|---|---|
+| 6 GB | IQ2_M / Q3_K_M | ~8–16K |
+| 8 GB | Q3_K_M / Q4_K_M | ~16–32K |
+| 12 GB | Q5_K_M / Q6_K | ~32K |
+| 16 GB+ | Q6_K / Q8_0 | ~32K (native) |
+
+## 🚀 How to run it
+```bash
+# ollama (pulls straight from this repo)
+ollama run hf.co/KikoCis/AMALIA-9B-0626-SFT-GGUF:Q5_K_M
+
+# llama.cpp
+llama-server -m amalia-9b-Q5_K_M.gguf -c 32768 --jinja
+llama-cli   -m amalia-9b-Q5_K_M.gguf -p "Explica em português o que é a saudade." -ngl 99
+```
+**Recommended sampling**: temperature ~0.7, top_p ~0.9. It's a **chat/instruct** model (uses its built-in chat template) — prompt it in **European Portuguese** for best results.
+
+## ⚠️ Good to know
+- **Strengths**: European-Portuguese fluency and instruction-following; one of the first open LLMs built in Portugal.
+- **Limits**: specialised for Portuguese — not a coding/agentic model, and not tuned for English. Low-bit quants (IQ2/Q3) trade real fidelity for size (see KLD).
+- Absolute perplexity looks high only because it's measured on raw text with an instruct model; the KLD table is the fair quant-quality signal.
+
+## 📊 Evaluation methodology (how the numbers were measured)
+- **What**: quantization **fidelity** vs the F16 GGUF — not a task benchmark. `llama-perplexity --kl-divergence` (KLD mean/p95/max, ΔPPL, top-1 agreement).
+- **Corpus**: European-Portuguese text (Wikipedia + diverse), **ctx 2048**, same corpus family used for the imatrix calibration.
+- **Reference**: the model's own **F16** GGUF (KLD = 0 by definition).
+- **Date**: 2026-07. *Caveat: relative fidelity ranking across quants of this model; not comparable across different models/corpora.*
+
+## 🔁 Provenance & reproducibility
+- **Scripts**: [`scripts/`](scripts) — exact convert → imatrix → quant → KLD commands.
+- **imatrix**: [`amalia-9b-pt.imatrix`](amalia-9b-pt.imatrix) — importance matrix (Portuguese calibration) used for the IQ/K quants.
+- **Checksums**: [`reports/artifact-sha256sums.txt`](reports/artifact-sha256sums.txt) — `shasum -a 256 -c` to verify.
+- **Source**: [`amalia-llm/AMALIA-9B-0626-SFT`](https://huggingface.co/amalia-llm/AMALIA-9B-0626-SFT) — weights **unmodified** (faithful quantization).
+
+## 🗒️ Changelog
+- 2026-07 v1: imatrix GGUF ladder (IQ2_M → Q8_0) + KLD/PPL metrics, Portuguese-calibrated.
+
+## 📚 Credit & license
+Model, weights, training data: **© the AMALIA-LLM project** — [model](https://huggingface.co/amalia-llm/AMALIA-9B-0626-SFT) · [website](https://amaliallm.pt/) · [paper (PROPOR 2026)](https://aclanthology.org/2026.propor-1.38/). Quant ladder + imatrix (pt) + KLD/PPL metrics: KikoCis. **Apache-2.0** (same as upstream). No weights modified.

amalia-9b-pt.imatrix

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1a919543f915131aca29bfd62b609ffc664a6e4d580921630558f6c21da87394
+size 6238400

assemble_card.py

@@ -0,0 +1,65 @@
+#!/usr/bin/env python3
+"""Fill the KLD table into README.md and render the two charts, from quant-summary.csv."""
+import csv, os, sys
+import matplotlib
+matplotlib.use("Agg")
+import matplotlib.pyplot as plt
+
+W = "/Users/kikocisneros/coco_ppl/aq_amalia"
+PUB = f"{W}/publish"
+CSV = f"{W}/metrics/quant-summary.csv"
+os.makedirs(f"{PUB}/metrics", exist_ok=True)
+
+rows = list(csv.DictReader(open(CSV)))
+def f(x, d=None):
+    try: return float(x)
+    except: return d
+
+# order big->small by size for the table
+order = sorted(rows, key=lambda r: -f(r["size_gb"], 0))
+
+# --- markdown KLD table ---
+hdr = "| Model | Size GB | PPL | ΔPPL vs F16 | KLD mean | KLD p95 | Top-1 match |\n|---|---:|---:|---:|---:|---:|---:|"
+lines = [hdr]
+for r in order:
+    name = r["name"]
+    ppl = r["ppl"]; dppl = r["ppl_delta_vs_f16"]
+    km = r["kld_mean"]; kp = r["kld_p95"]; t1 = r["top1_match_pct"]
+    if name == "F16":
+        lines.append(f"| **F16 (reference)** | {f(r['size_gb']):.1f} | {f(ppl):.2f} | 0.000 | 0.0000 | 0.0000 | 100.0% |")
+    else:
+        dv = f(dppl, 0); sign = "+" if dv >= 0 else ""
+        lines.append(f"| {name} | {f(r['size_gb']):.2f} | {f(ppl):.2f} | {sign}{dv:.3f} | {f(km):.4f} | {f(kp):.4f} | {f(t1):.2f}% |")
+table = "\n".join(lines)
+
+readme = open(f"{PUB}/README.md").read()
+readme = readme.replace("<!--KLD_TABLE_PLACEHOLDER-->", table)
+open(f"{PUB}/README.md", "w").write(readme)
+print("README table filled:\n" + table)
+
+# --- chart 1: KLD mean vs size ---
+q = [r for r in order if r["name"] != "F16"]
+q = sorted(q, key=lambda r: f(r["size_gb"]))
+sizes = [f(r["size_gb"]) for r in q]; klds = [f(r["kld_mean"]) for r in q]; names = [r["name"] for r in q]
+plt.figure(figsize=(7,4.2))
+plt.plot(sizes, klds, "o-", color="#2ca02c", lw=2, ms=7)
+for x,y,n in zip(sizes,klds,names):
+    plt.annotate(n, (x,y), textcoords="offset points", xytext=(6,6), fontsize=8)
+plt.yscale("log"); plt.xlabel("file size (GB)"); plt.ylabel("KLD mean (nats, log) — lower = closer to F16")
+plt.title("AMALIA-9B GGUF — fidelity vs size (KLD vs F16, pt, ctx 2048)")
+plt.grid(True, which="both", alpha=0.3); plt.tight_layout()
+plt.savefig(f"{PUB}/metrics/chart-kld-vs-size.png", dpi=130); plt.close()
+
+# --- chart 2: top-1 match ---
+plt.figure(figsize=(7,4.2))
+t1s = [f(r["top1_match_pct"]) for r in q]
+bars = plt.bar(names, t1s, color="#1f77b4")
+for b,v in zip(bars,t1s): plt.text(b.get_x()+b.get_width()/2, v+0.4, f"{v:.1f}", ha="center", fontsize=8)
+plt.ylabel("Top-1 token match vs F16 (%)"); plt.ylim(min(t1s)-3, 100.5)
+plt.title("AMALIA-9B GGUF — top-1 agreement with F16")
+plt.grid(True, axis="y", alpha=0.3); plt.tight_layout()
+plt.savefig(f"{PUB}/metrics/chart-top1.png", dpi=130); plt.close()
+
+# copy the csv into publish
+import shutil; shutil.copy(CSV, f"{PUB}/metrics/quant-summary.csv")
+print("charts + csv written to", f"{PUB}/metrics/")

metrics/quant-summary-ctx512.csv

@@ -0,0 +1,8 @@
+name,size_gb,ppl,ppl_delta_vs_f16,kld_mean,kld_p95,kld_max,top1_match_pct
+F16,17.05,4.44,0.0,0.0,0.0,0.0,100.0
+Q8_0,9.06,32.115409,0.091330,0.009722,0.031179,3.845082,95.497
+Q6_K,7.00,31.491478,-0.532602,0.088915,0.260961,16.998377,86.758
+Q5_K_M,6.07,31.717572,-0.306507,0.080225,0.280754,6.408488,86.025
+Q4_K_M,5.20,32.896111,0.872032,0.206477,0.734504,13.929957,78.730
+IQ4_XS,4.70,30.450651,-1.573428,0.384149,1.645543,30.692392,76.901
+Q3_K_M,4.24,35.781539,3.757460,0.420631,1.423969,32.735237,70.642

metrics/quant-summary.csv

@@ -0,0 +1,9 @@
+name,size_gb,ppl,ppl_delta_vs_f16,kld_mean,kld_p95,kld_max,top1_match_pct
+F16,17.05,20.0376,0.0,0.0,0.0,0.0,100.0
+Q8_0,9.06,20.200587,0.177055,0.005708,0.019779,1.475488,96.381
+Q6_K,7.00,19.932671,-0.090860,0.077855,0.174014,16.764179,89.056
+Q5_K_M,6.08,19.797018,-0.226513,0.051125,0.174651,9.869919,88.887
+Q4_K_M,5.20,20.226883,0.203352,0.222994,0.742914,21.454760,81.387
+IQ4_XS,4.70,20.780607,0.757076,0.217275,0.760046,17.964737,80.554
+Q3_K_M,4.25,22.238185,2.214654,0.325828,1.148634,19.564314,73.580
+IQ2_M,3.11,21.991922,1.968390,0.821109,2.905154,27.317856,60.690

reports/artifact-sha256sums.txt

@@ -0,0 +1,5 @@
+c653273b579c7a8ae813389db68178f3520fff2735aee6a4799f194ddef29b6d  amalia-9b-IQ2_M.gguf
+6111748cecc0d26478194412cc34f0eb63f91b1b62098755298019f868b503b1  amalia-9b-IQ4_XS.gguf
+2187c384a1d879825e3fab47867530093765dd5b1509393b179e73dc332302ab  amalia-9b-Q3_K_M.gguf
+e1be526dea752b1cd8e019d0b6444c9cb76f812a37e1f9bc309360923d6a1718  amalia-9b-Q4_K_M.gguf
+9487ae725635c5ca6b7c4e3768ca8bd9e28adcb2e88ff0a54d6e87250e999368  amalia-9b-Q5_K_M.gguf

reports/base2048.txt

@@ -0,0 +1,10 @@
+0.07.412.398 I 
+0.07.412.417 I system_info: n_threads = 12 (n_threads_batch = 12) / 16 | MTL : EMBED_LIBRARY = 1 | CPU : NEON = 1 | ARM_FMA = 1 | FP16_VA = 1 | MATMUL_INT8 = 1 | DOTPROD = 1 | SME = 1 | ACCELERATE = 1 | OPENMP = 1 | REPACK = 1 | 
+0.07.412.501 I perplexity: saving all logits to /Users/kikocisneros/coco_ppl/aq_amalia/kld_base_2048.dat
+0.07.412.504 I perplexity: tokenizing the input ..
+0.07.557.142 I perplexity: tokenization took 144.636 ms
+0.07.557.200 I perplexity: calculating perplexity over 44 chunks, n_ctx=2048, batch_size=2048, n_seq=1
+0.10.211.335 I perplexity: 2.63 seconds per pass - ETA 1.92 minutes
+[1]39.2953,[2]43.3171,[3]30.9450,[4]21.2218,[5]16.7458,[6]18.9088,[7]21.3953,[8]23.6025,[9]20.7184,[10]23.2438,[11]20.7743,[12]19.9841,[13]19.8370,[14]21.0076,[15]20.2485,[16]18.7585,[17]18.6742,[18]19.2116,[19]19.3664,[20]19.7867,[21]19.9371,[22]19.1936,[23]18.1903,[24]18.3383,[25]17.6663,[26]17.9583,[27]17.1198,[28]17.3306,[29]17.6339,[30]18.0336,[31]17.8652,[32]17.6116,[33]17.8595,[34]17.9006,[35]18.1363,[36]18.6202,[37]18.8459,[38]19.2546,[39]19.4834,[40]19.5976,[41]19.8347,[42]20.0916,[43]20.3915,[44]20.0376,
+3.42.832.788 I Final estimate: PPL = 20.0376 +/- 0.28573
+

reports/kld2048-IQ4_XS.txt

@@ -0,0 +1,91 @@
+0.02.265.152 I 
+0.02.265.171 I system_info: n_threads = 12 (n_threads_batch = 12) / 16 | MTL : EMBED_LIBRARY = 1 | CPU : NEON = 1 | ARM_FMA = 1 | FP16_VA = 1 | MATMUL_INT8 = 1 | DOTPROD = 1 | SME = 1 | ACCELERATE = 1 | OPENMP = 1 | REPACK = 1 | 
+0.02.282.825 I kl_divergence: computing over 44 chunks, n_ctx=2048, batch_size=2048, n_seq=1
+0.06.574.032 I kl_divergence: 4.29 seconds per pass - ETA 3.13 minutes
+
+chunk             PPL               ln(PPL(Q)/PPL(base))          KL Divergence              Δp RMS            Same top p
+   1      40.3335 ±    3.9997       0.02855 ±    0.02025       0.13629 ±    0.00600     8.309 ±  0.525 %    77.126 ±  1.314 %
+   2      45.6377 ±    3.2447       0.05342 ±    0.01545       0.14017 ±    0.00452     8.435 ±  0.420 %    77.859 ±  0.918 %
+   3      32.5277 ±    1.9038       0.05070 ±    0.01138       0.11585 ±    0.00354     7.950 ±  0.385 %    81.167 ±  0.706 %
+   4      22.3582 ±    1.1201       0.05278 ±    0.00981       0.10672 ±    0.00309     7.709 ±  0.320 %    83.431 ±  0.581 %
+   5      17.5481 ±    0.7691       0.04729 ±    0.00824       0.09422 ±    0.00274     7.386 ±  0.299 %    85.357 ±  0.494 %
+   6      19.9353 ±    0.8053       0.05327 ±    0.00787       0.10348 ±    0.00253     7.645 ±  0.260 %    84.229 ±  0.465 %
+   7      22.4908 ±    0.8507       0.05041 ±    0.00738       0.10800 ±    0.00232     7.619 ±  0.231 %    83.215 ±  0.442 %
+   8      24.8056 ±    0.8778       0.05013 ±    0.00704       0.11587 ±    0.00227     7.686 ±  0.216 %    82.270 ±  0.422 %
+   9      21.6580 ±    0.7181       0.04474 ±    0.00654       0.11110 ±    0.00216     7.668 ±  0.205 %    83.089 ±  0.391 %
+  10      24.0176 ±    0.7624       0.03313 ±    0.00628       0.11687 ±    0.00205     7.632 ±  0.191 %    82.356 ±  0.377 %
+  11      21.4310 ±    0.6462       0.03173 ±    0.00596       0.11504 ±    0.00200     7.738 ±  0.182 %    83.009 ±  0.354 %
+  12      20.6137 ±    0.5894       0.03158 ±    0.00565       0.11387 ±    0.00188     7.819 ±  0.170 %    83.187 ±  0.338 %
+  13      20.3602 ±    0.5565       0.02655 ±    0.00542       0.11427 ±    0.00178     7.884 ±  0.161 %    83.021 ±  0.326 %
+  14      21.6156 ±    0.5723       0.02909 ±    0.00524       0.11711 ±    0.00173     7.871 ±  0.154 %    82.733 ±  0.316 %
+  15      19.3184 ±    0.4889      -0.04649 ±    0.00697       0.24254 ±    0.00679    12.238 ±  0.256 %    80.932 ±  0.317 %
+  16      18.0005 ±    0.4389      -0.04076 ±    0.00660       0.23150 ±    0.00639    12.002 ±  0.246 %    81.488 ±  0.304 %
+  17      18.1333 ±    0.4253      -0.02893 ±    0.00634       0.22577 ±    0.00603    11.875 ±  0.235 %    81.375 ±  0.295 %
+  18      18.7734 ±    0.4274      -0.02264 ±    0.00605       0.21874 ±    0.00570    11.668 ±  0.227 %    81.378 ±  0.287 %
+  19      19.0545 ±    0.4212      -0.01582 ±    0.00583       0.21391 ±    0.00541    11.529 ±  0.218 %    81.335 ±  0.279 %
+  20      19.5749 ±    0.4208      -0.01036 ±    0.00563       0.21026 ±    0.00515    11.357 ±  0.211 %    81.271 ±  0.273 %
+  21      19.7828 ±    0.4143      -0.00736 ±    0.00541       0.20494 ±    0.00491    11.190 ±  0.204 %    81.362 ±  0.266 %
+  22      19.0823 ±    0.3883      -0.00542 ±    0.00520       0.19826 ±    0.00469    11.031 ±  0.198 %    81.707 ±  0.258 %
+  23      18.1130 ±    0.3588      -0.00388 ±    0.00502       0.19222 ±    0.00450    10.905 ±  0.193 %    82.082 ±  0.250 %
+  24      18.3864 ±    0.3579       0.00334 ±    0.00491       0.19097 ±    0.00433    10.892 ±  0.187 %    81.953 ±  0.245 %
+  25      17.7117 ±    0.3365       0.00325 ±    0.00474       0.18507 ±    0.00416    10.719 ±  0.183 %    82.311 ±  0.239 %
+  26      18.0524 ±    0.3363       0.00589 ±    0.00461       0.18246 ±    0.00401    10.646 ±  0.178 %    82.209 ±  0.234 %
+  27      17.2297 ±    0.3139       0.00719 ±    0.00448       0.17912 ±    0.00388    10.595 ±  0.173 %    82.448 ±  0.229 %
+  28      17.5078 ±    0.3130       0.01093 ±    0.00437       0.17648 ±    0.00375    10.505 ±  0.169 %    82.488 ±  0.225 %
+  29      17.8562 ±    0.3150       0.01326 ±    0.00430       0.17700 ±    0.00363    10.494 ±  0.165 %    82.317 ±  0.222 %
+  30      18.2704 ±    0.3167       0.01375 ±    0.00422       0.17611 ±    0.00352    10.429 ±  0.161 %    82.199 ±  0.218 %
+  31      18.1377 ±    0.3079       0.01583 ±    0.00412       0.17346 ±    0.00341    10.356 ±  0.157 %    82.304 ±  0.214 %
+  32      18.4414 ±    0.3083       0.04679 ±    0.00471       0.21744 ±    0.00391    12.102 ±  0.173 %    81.137 ±  0.216 %
+  33      18.7050 ±    0.3074       0.04699 ±    0.00460       0.21434 ±    0.00379    11.998 ±  0.169 %    81.089 ±  0.213 %
+  34      18.7511 ±    0.3029       0.04714 ±    0.00449       0.21119 ±    0.00368    11.880 ±  0.166 %    81.045 ±  0.210 %
+  35      19.0228 ±    0.3036       0.04844 ±    0.00441       0.20948 ��    0.00359    11.800 ±  0.162 %    81.022 ±  0.207 %
+  36      19.5390 ±    0.3089       0.04896 ±    0.00435       0.20948 ±    0.00350    11.746 ±  0.159 %    80.854 ±  0.205 %
+  37      19.7770 ±    0.3082       0.04900 ±    0.00425       0.20617 ±    0.00340    11.636 ±  0.157 %    80.954 ±  0.202 %
+  38      20.2114 ±    0.3112       0.04925 ±    0.00417       0.20401 ±    0.00332    11.547 ±  0.154 %    80.905 ±  0.199 %
+  39      20.4638 ±    0.3109       0.04984 ±    0.00408       0.20165 ±    0.00323    11.452 ±  0.151 %    80.851 ±  0.197 %
+  40      20.5837 ±    0.3085       0.04982 ±    0.00401       0.19926 ±    0.00316    11.368 ±  0.149 %    80.880 ±  0.194 %
+  41      20.8094 ±    0.3084       0.04872 ±    0.00394       0.19737 ±    0.00308    11.293 ±  0.146 %    80.876 ±  0.192 %
+  42      21.0830 ±    0.3090       0.04890 ±    0.00388       0.19638 ±    0.00301    11.238 ±  0.144 %    80.836 ±  0.190 %
+  43      21.3715 ±    0.3096       0.04766 ±    0.00381       0.19445 ±    0.00295    11.163 ±  0.142 %    80.820 ±  0.188 %
+  44      20.9837 ±    0.2996       0.04684 ±    0.00373       0.19052 ±    0.00288    11.048 ±  0.140 %    81.061 ±  0.185 %
+
+====== Perplexity statistics ======
+Mean PPL(Q)                   :  20.983665 ±   0.299595
+Mean PPL(base)                :  20.023532 ±   0.285215
+Cor(ln(PPL(Q)), ln(PPL(base))):  96.57%
+Mean ln(PPL(Q)/PPL(base))     :   0.046836 ±   0.003733
+Mean PPL(Q)/PPL(base)         :   1.047950 ±   0.003912
+Mean PPL(Q)-PPL(base)         :   0.960133 ±   0.077854
+
+====== KL divergence statistics ======
+Mean    KLD:   0.190517 ±   0.002882
+Maximum KLD:  15.982407
+99.9%   KLD:   8.710998
+99.0%   KLD:   2.718909
+95.0%   KLD:   0.599236
+90.0%   KLD:   0.322753
+Median  KLD:   0.065426
+10.0%   KLD:   0.003866
+ 5.0%   KLD:   0.000702
+ 1.0%   KLD:   0.000018
+ 0.1%   KLD:   0.000000
+Minimum KLD:  -0.000021
+
+====== Token probability statistics ======
+Mean    Δp: -0.745 ± 0.052 %
+Maximum Δp: 99.976%
+99.9%   Δp: 90.469%
+99.0%   Δp: 30.855%
+95.0%   Δp: 10.810%
+90.0%   Δp:  5.319%
+75.0%   Δp:  0.652%
+Median  Δp: -0.007%
+25.0%   Δp: -1.643%
+10.0%   Δp: -7.913%
+ 5.0%   Δp: -14.614%
+ 1.0%   Δp: -38.569%
+ 0.1%   Δp: -92.396%
+Minimum Δp: -99.956%
+RMS Δp    : 11.048 ± 0.140 %
+Same top p: 81.061 ± 0.185 %
+

reports/kld2048-Q3_K_M.txt

@@ -0,0 +1,91 @@
+0.02.111.565 I 
+0.02.111.584 I system_info: n_threads = 12 (n_threads_batch = 12) / 16 | MTL : EMBED_LIBRARY = 1 | CPU : NEON = 1 | ARM_FMA = 1 | FP16_VA = 1 | MATMUL_INT8 = 1 | DOTPROD = 1 | SME = 1 | ACCELERATE = 1 | OPENMP = 1 | REPACK = 1 | 
+0.02.126.312 I kl_divergence: computing over 44 chunks, n_ctx=2048, batch_size=2048, n_seq=1
+0.05.780.560 I kl_divergence: 3.65 seconds per pass - ETA 2.67 minutes
+
+chunk             PPL               ln(PPL(Q)/PPL(base))          KL Divergence              Δp RMS            Same top p
+   1      42.7479 ±    4.1534       0.08668 ±    0.03082       0.29717 ±    0.01140    11.898 ±  0.638 %    70.479 ±  1.427 %
+   2      47.6359 ±    3.3896       0.09627 ±    0.02228       0.29492 ±    0.00839    11.628 ±  0.483 %    71.065 ±  1.003 %
+   3      33.5703 ±    1.9598       0.08225 ±    0.01670       0.25226 ±    0.00658    10.990 ±  0.407 %    74.389 ±  0.788 %
+   4      22.5348 ±    1.1245       0.06065 ±    0.01429       0.23519 ±    0.00605    11.056 ±  0.355 %    76.931 ±  0.659 %
+   5      17.7061 ±    0.7722       0.05625 ±    0.01228       0.20981 ±    0.00535    10.936 ±  0.328 %    79.277 ±  0.567 %
+   6      20.3186 ±    0.8182       0.07232 ±    0.01141       0.23402 ±    0.00516    11.256 ±  0.293 %    77.566 ±  0.532 %
+   7      23.0720 ±    0.8697       0.07592 ±    0.01066       0.24514 ±    0.00478    11.324 ±  0.269 %    76.288 ±  0.503 %
+   8      25.5785 ±    0.9040       0.08081 ±    0.01008       0.25942 ±    0.00458    11.354 ±  0.247 %    75.134 ±  0.478 %
+   9      22.3535 ±    0.7405       0.07635 ±    0.00952       0.25430 ±    0.00453    11.657 ±  0.254 %    76.116 ±  0.444 %
+  10      25.1425 ±    0.8005       0.07891 ±    0.00903       0.25970 ±    0.00421    11.540 ±  0.238 %    75.034 ±  0.428 %
+  11      22.3257 ±    0.6730       0.07263 ±    0.00862       0.25585 ±    0.00412    11.767 ±  0.232 %    75.873 ±  0.403 %
+  12      21.6625 ±    0.6205       0.08121 ±    0.00819       0.25117 ±    0.00386    11.785 ±  0.217 %    76.303 ±  0.384 %
+  13      21.7052 ±    0.5958       0.09052 ±    0.00794       0.26004 ±    0.00373    12.010 ±  0.205 %    75.652 ±  0.372 %
+  14      23.0156 ±    0.6114       0.09185 ±    0.00785       0.27259 ±    0.00379    12.104 ±  0.197 %    74.843 ±  0.363 %
+  15      22.1689 ±    0.5661       0.09114 ±    0.00759       0.27006 ±    0.00361    12.263 ±  0.190 %    74.963 ±  0.350 %
+  16      20.4489 ±    0.5031       0.08678 ±    0.00731       0.26437 ±    0.00360    12.378 ±  0.189 %    75.611 ±  0.336 %
+  17      20.2855 ±    0.4812       0.08323 ±    0.00710       0.26529 ±    0.00345    12.346 ±  0.180 %    75.585 ±  0.326 %
+  18      20.9329 ±    0.4821       0.08625 ±    0.00683       0.26274 ±    0.00329    12.240 ±  0.173 %    75.377 ±  0.317 %
+  19      21.2465 ±    0.4762       0.09307 ±    0.00670       0.26558 ±    0.00319    12.262 ±  0.167 %    75.217 ±  0.310 %
+  20      21.7338 ±    0.4741       0.09425 ±    0.00656       0.26955 ±    0.00311    12.232 ±  0.161 %    74.961 ±  0.303 %
+  21      22.1211 ±    0.4723       0.10436 ±    0.00643       0.27138 ±    0.00303    12.162 ±  0.156 %    74.910 ±  0.296 %
+  22      21.2445 ±    0.4404       0.10191 ±    0.00624       0.26645 ±    0.00300    12.110 ±  0.153 %    75.260 ±  0.288 %
+  23      20.0829 ±    0.4053       0.09935 ±    0.00604       0.26110 ±    0.00291    12.072 ±  0.150 %    75.643 ±  0.280 %
+  24      20.2131 ±    0.4001       0.09806 ±    0.00595       0.26261 ±    0.00286    12.108 ±  0.146 %    75.497 ±  0.274 %
+  25      19.4393 ±    0.3756       0.09633 ±    0.00575       0.25676 ±    0.00277    12.003 ±  0.143 %    75.937 ±  0.267 %
+  26      19.7562 ±    0.3738       0.09608 ±    0.00561       0.25453 ±    0.00267    11.947 ±  0.139 %    75.874 ±  0.262 %
+  27      18.7920 ±    0.3474       0.09399 ±    0.00549       0.25243 ±    0.00264    11.995 ±  0.137 %    76.178 ±  0.256 %
+  28      19.0032 ±    0.3443       0.09289 ±    0.00537       0.25198 ±    0.00257    11.975 ±  0.134 %    76.110 ±  0.252 %
+  29      19.3639 ±    0.3458       0.09432 ±    0.00529       0.25422 ±    0.00253    12.049 ±  0.132 %    75.953 ±  0.248 %
+  30      19.8213 ±    0.3481       0.09523 ±    0.00523       0.25674 ±    0.00250    12.017 ±  0.129 %    75.751 ±  0.245 %
+  31      19.6233 ±    0.3376       0.09455 ±    0.00513       0.25581 ±    0.00245    12.030 ±  0.127 %    75.707 ±  0.241 %
+  32      19.9074 ±    0.3375       0.12328 ±    0.00546       0.29033 ±    0.00299    13.298 ±  0.145 %    74.875 ±  0.240 %
+  33      20.1685 ±    0.3361       0.12232 ±    0.00537       0.28906 ±    0.00292    13.241 ±  0.142 %    74.866 ±  0.236 %
+  34      20.2095 ±    0.3309       0.12204 ±    0.00526       0.28718 ±    0.00285    13.203 ±  0.139 %    74.780 ±  0.233 %
+  35      20.4523 ±    0.3306       0.12090 ±    0.00520       0.28946 ��    0.00280    13.230 ±  0.136 %    74.629 ±  0.230 %
+  36      20.7814 ±    0.3325       0.11060 ±    0.00550       0.33851 ±    0.00368    13.804 ±  0.138 %    73.727 ±  0.229 %
+  37      21.0618 ±    0.3323       0.11193 ±    0.00539       0.33515 ±    0.00359    13.719 ±  0.136 %    73.731 ±  0.226 %
+  38      21.5154 ±    0.3356       0.11177 ±    0.00530       0.33381 ±    0.00351    13.659 ±  0.133 %    73.661 ±  0.223 %
+  39      21.8036 ±    0.3358       0.11326 ±    0.00520       0.33132 ±    0.00343    13.582 ±  0.131 %    73.697 ±  0.220 %
+  40      21.9536 ±    0.3338       0.11425 ±    0.00512       0.32989 ±    0.00335    13.535 ±  0.129 %    73.678 ±  0.218 %
+  41      22.2466 ±    0.3346       0.11551 ±    0.00504       0.32881 ±    0.00328    13.482 ±  0.127 %    73.619 ±  0.215 %
+  42      22.5787 ±    0.3362       0.11744 ±    0.00498       0.32853 ±    0.00322    13.436 ±  0.125 %    73.542 ±  0.213 %
+  43      22.9255 ±    0.3376       0.11785 ±    0.00491       0.32810 ±    0.00317    13.396 ±  0.124 %    73.505 ±  0.210 %
+  44      22.4858 ±    0.3263       0.11597 ±    0.00481       0.32229 ±    0.00311    13.278 ±  0.122 %    73.760 ±  0.207 %
+
+====== Perplexity statistics ======
+Mean PPL(Q)                   :  22.485774 ±   0.326277
+Mean PPL(base)                :  20.023532 ±   0.285215
+Cor(ln(PPL(Q)), ln(PPL(base))):  94.42%
+Mean ln(PPL(Q)/PPL(base))     :   0.115975 ±   0.004810
+Mean PPL(Q)/PPL(base)         :   1.122967 ±   0.005402
+Mean PPL(Q)-PPL(base)         :   2.462243 ±   0.109870
+
+====== KL divergence statistics ======
+Mean    KLD:   0.322294 ±   0.003108
+Maximum KLD:  15.541307
+99.9%   KLD:   7.824029
+99.0%   KLD:   3.186437
+95.0%   KLD:   1.145174
+90.0%   KLD:   0.691000
+Median  KLD:   0.148998
+10.0%   KLD:   0.009049
+ 5.0%   KLD:   0.001746
+ 1.0%   KLD:   0.000052
+ 0.1%   KLD:   0.000001
+Minimum KLD:  -0.000018
+
+====== Token probability statistics ======
+Mean    Δp: -1.378 ± 0.062 %
+Maximum Δp: 99.759%
+99.9%   Δp: 78.834%
+99.0%   Δp: 36.834%
+95.0%   Δp: 15.041%
+90.0%   Δp:  7.773%
+75.0%   Δp:  0.964%
+Median  Δp: -0.017%
+25.0%   Δp: -2.497%
+10.0%   Δp: -12.388%
+ 5.0%   Δp: -22.110%
+ 1.0%   Δp: -50.488%
+ 0.1%   Δp: -92.557%
+Minimum Δp: -99.918%
+RMS Δp    : 13.278 ± 0.122 %
+Same top p: 73.760 ± 0.207 %
+

reports/kld2048-Q4_K_M.txt

@@ -0,0 +1,91 @@
+0.02.444.999 I 
+0.02.445.022 I system_info: n_threads = 12 (n_threads_batch = 12) / 16 | MTL : EMBED_LIBRARY = 1 | CPU : NEON = 1 | ARM_FMA = 1 | FP16_VA = 1 | MATMUL_INT8 = 1 | DOTPROD = 1 | SME = 1 | ACCELERATE = 1 | OPENMP = 1 | REPACK = 1 | 
+0.02.458.394 I kl_divergence: computing over 44 chunks, n_ctx=2048, batch_size=2048, n_seq=1
+0.06.572.750 I kl_divergence: 4.11 seconds per pass - ETA 3.02 minutes
+
+chunk             PPL               ln(PPL(Q)/PPL(base))          KL Divergence              Δp RMS            Same top p
+   1      41.2795 ±    4.1398       0.05173 ±    0.02221       0.14531 ±    0.00574     8.854 ±  0.552 %    77.126 ±  1.314 %
+   2      45.0236 ±    3.2155       0.03987 ±    0.01546       0.14469 ±    0.00415     8.361 ±  0.366 %    78.250 ±  0.912 %
+   3      31.5781 ±    1.8364       0.02107 ±    0.01123       0.11655 ±    0.00343     7.819 ±  0.360 %    82.079 ±  0.692 %
+   4      21.6469 ±    1.0758       0.02045 ±    0.00935       0.10766 ±    0.00419     7.739 ±  0.332 %    83.993 ±  0.573 %
+   5      16.9931 ±    0.7377       0.01515 ±    0.00792       0.09377 ±    0.00344     7.450 ±  0.295 %    85.885 ±  0.487 %
+   6      19.4869 ±    0.7820       0.03052 ±    0.00775       0.10659 ±    0.00321     7.721 ±  0.261 %    84.653 ±  0.460 %
+   7      22.6632 ±    0.8578       0.05804 ±    0.00758       0.11565 ±    0.00297     7.946 ±  0.239 %    83.550 ±  0.438 %
+   8      20.9572 ±    0.7326      -0.11846 ±    0.01231       0.39262 ±    0.01300    14.625 ±  0.373 %    78.434 ±  0.455 %
+   9      18.5839 ±    0.6074      -0.10834 ±    0.01109       0.35627 ±    0.01162    14.009 ±  0.348 %    79.700 ±  0.419 %
+  10      21.2826 ±    0.6718      -0.08776 ±    0.01025       0.33616 ±    0.01050    13.492 ±  0.328 %    79.413 ±  0.400 %
+  11      19.1917 ±    0.5743      -0.07863 ±    0.00945       0.31249 ±    0.00958    13.037 ±  0.309 %    80.450 ±  0.374 %
+  12      19.5849 ±    0.5619      -0.01962 ±    0.00936       0.32527 ±    0.00897    13.301 ±  0.289 %    79.822 ±  0.362 %
+  13      19.5401 ±    0.5360      -0.01456 ±    0.00876       0.31061 ±    0.00831    13.029 ±  0.274 %    79.946 ±  0.347 %
+  14      20.8731 ±    0.5551      -0.00586 ±    0.00827       0.30007 ±    0.00774    12.756 ±  0.261 %    79.807 ±  0.335 %
+  15      20.1689 ±    0.5160      -0.00341 ±    0.00783       0.28704 ±    0.00724    12.577 ±  0.249 %    80.137 ±  0.322 %
+  16      18.6731 ±    0.4595      -0.00407 ±    0.00741       0.27279 ±    0.00682    12.324 ±  0.240 %    80.816 ±  0.308 %
+  17      18.6061 ±    0.4410      -0.00319 ±    0.00707       0.26420 ±    0.00643    12.100 ±  0.231 %    80.823 ±  0.299 %
+  18      19.1916 ±    0.4411      -0.00060 ±    0.00674       0.25503 ±    0.00608    11.886 ±  0.223 %    80.884 ±  0.290 %
+  19      19.4546 ±    0.4344       0.00496 ±    0.00649       0.24983 ±    0.00577    11.775 ±  0.214 %    80.794 ±  0.283 %
+  20      19.9924 ±    0.4346       0.01074 ±    0.00625       0.24438 ±    0.00549    11.600 ±  0.207 %    80.689 ±  0.276 %
+  21      20.2596 ±    0.4294       0.01645 ±    0.00602       0.23860 ±    0.00524    11.452 ±  0.200 %    80.752 ±  0.269 %
+  22      19.5131 ±    0.4014       0.01690 ±    0.00577       0.23014 ±    0.00501    11.276 ±  0.195 %    81.112 ±  0.261 %
+  23      18.4590 ±    0.3693       0.01504 ±    0.00555       0.22207 ±    0.00480    11.100 ±  0.190 %    81.550 ±  0.253 %
+  24      18.6444 ±    0.3663       0.01727 ±    0.00542       0.21940 ±    0.00462    11.020 ±  0.184 %    81.496 ±  0.248 %
+  25      17.9511 ±    0.3440       0.01668 ±    0.00522       0.21212 ±    0.00444    10.845 ±  0.180 %    81.896 ±  0.241 %
+  26      18.2686 ±    0.3430       0.01779 ±    0.00506       0.20778 ±    0.00428    10.719 ±  0.175 %    81.908 ±  0.236 %
+  27      17.4065 ±    0.3197       0.01740 ±    0.00493       0.20349 ±    0.00413    10.692 ±  0.171 %    82.206 ±  0.230 %
+  28      17.6385 ±    0.3178       0.01837 ±    0.00479       0.20009 ±    0.00399    10.591 ±  0.167 %    82.216 ±  0.226 %
+  29      18.0042 ±    0.3199       0.02152 ±    0.00468       0.19849 ±    0.00386    10.544 ±  0.162 %    82.149 ±  0.222 %
+  30      18.4178 ±    0.3215       0.02179 ±    0.00459       0.19651 ±    0.00374    10.470 ±  0.159 %    82.030 ±  0.219 %
+  31      18.2709 ±    0.3124       0.02314 ±    0.00447       0.19322 ±    0.00362    10.412 ±  0.155 %    82.121 ±  0.215 %
+  32      17.8423 ±    0.2996       0.01376 ±    0.00451       0.19735 ±    0.00358    10.703 ±  0.155 %    82.005 ±  0.212 %
+  33      18.1069 ±    0.2990       0.01449 ±    0.00441       0.19455 ±    0.00348    10.613 ±  0.152 %    82.014 ±  0.209 %
+  34      18.1601 ±    0.2946       0.01511 ±    0.00432       0.19198 ±    0.00338    10.562 ±  0.148 %    82.014 ±  0.206 %
+  35      18.4193 ±    0.2951       0.01620 ±    0.00424       0.19046 ��    0.00329    10.519 ±  0.145 %    81.997 ±  0.203 %
+  36      18.9312 ±    0.3005       0.01736 ±    0.00433       0.20369 ±    0.00333    10.673 ±  0.142 %    81.435 ±  0.203 %
+  37      19.1886 ±    0.3005       0.01879 ±    0.00423       0.20058 ±    0.00325    10.580 ±  0.140 %    81.549 ±  0.199 %
+  38      19.5951 ±    0.3033       0.01828 ±    0.00415       0.19855 ±    0.00317    10.517 ±  0.138 %    81.535 ±  0.197 %
+  39      19.8455 ±    0.3031       0.01916 ±    0.00406       0.19609 ±    0.00309    10.454 ±  0.135 %    81.542 ±  0.194 %
+  40      19.9566 ±    0.3008       0.01888 ±    0.00399       0.19379 ±    0.00301    10.382 ±  0.133 %    81.569 ±  0.192 %
+  41      20.2158 ±    0.3012       0.01978 ±    0.00392       0.19213 ±    0.00294    10.323 ±  0.131 %    81.480 ±  0.190 %
+  42      20.5107 ±    0.3025       0.02138 ±    0.00386       0.19120 ±    0.00288    10.284 ±  0.128 %    81.434 ±  0.188 %
+  43      20.8351 ±    0.3039       0.02224 ±    0.00379       0.19001 ±    0.00282    10.229 ±  0.126 %    81.395 ±  0.186 %
+  44      20.4607 ±    0.2940       0.02160 ±    0.00371       0.18611 ±    0.00276    10.126 ±  0.125 %    81.632 ±  0.183 %
+
+====== Perplexity statistics ======
+Mean PPL(Q)                   :  20.460742 ±   0.294034
+Mean PPL(base)                :  20.023532 ±   0.285215
+Cor(ln(PPL(Q)), ln(PPL(base))):  96.64%
+Mean ln(PPL(Q)/PPL(base))     :   0.021600 ±   0.003713
+Mean PPL(Q)/PPL(base)         :   1.021835 ±   0.003794
+Mean PPL(Q)-PPL(base)         :   0.437210 ±   0.075617
+
+====== KL divergence statistics ======
+Mean    KLD:   0.186110 ±   0.002757
+Maximum KLD:  18.745331
+99.9%   KLD:   8.276571
+99.0%   KLD:   2.376208
+95.0%   KLD:   0.638440
+90.0%   KLD:   0.343436
+Median  KLD:   0.062474
+10.0%   KLD:   0.003218
+ 5.0%   KLD:   0.000575
+ 1.0%   KLD:   0.000016
+ 0.1%   KLD:   0.000000
+Minimum KLD:  -0.000008
+
+====== Token probability statistics ======
+Mean    Δp: -0.062 ± 0.048 %
+Maximum Δp: 99.658%
+99.9%   Δp: 90.783%
+99.0%   Δp: 34.222%
+95.0%   Δp: 11.752%
+90.0%   Δp:  5.987%
+75.0%   Δp:  0.861%
+Median  Δp: -0.003%
+25.0%   Δp: -1.287%
+10.0%   Δp: -7.002%
+ 5.0%   Δp: -12.914%
+ 1.0%   Δp: -30.396%
+ 0.1%   Δp: -62.846%
+Minimum Δp: -98.875%
+RMS Δp    : 10.126 ± 0.125 %
+Same top p: 81.632 ± 0.183 %
+

reports/kld2048-Q5_K_M.txt

@@ -0,0 +1,91 @@
+0.02.768.224 I 
+0.02.768.244 I system_info: n_threads = 12 (n_threads_batch = 12) / 16 | MTL : EMBED_LIBRARY = 1 | CPU : NEON = 1 | ARM_FMA = 1 | FP16_VA = 1 | MATMUL_INT8 = 1 | DOTPROD = 1 | SME = 1 | ACCELERATE = 1 | OPENMP = 1 | REPACK = 1 | 
+0.02.782.686 I kl_divergence: computing over 44 chunks, n_ctx=2048, batch_size=2048, n_seq=1
+0.07.371.405 I kl_divergence: 4.59 seconds per pass - ETA 3.35 minutes
+
+chunk             PPL               ln(PPL(Q)/PPL(base))          KL Divergence              Δp RMS            Same top p
+   1      38.8757 ±    3.9445      -0.00827 ±    0.01320       0.05545 ±    0.00215     5.215 ±  0.340 %    85.337 ±  1.107 %
+   2      42.6337 ±    3.0419      -0.01467 ±    0.01052       0.06207 ±    0.00192     5.466 ±  0.256 %    85.679 ±  0.775 %
+   3      30.5964 ±    1.7826      -0.01051 ±    0.00743       0.04750 ±    0.00140     4.847 ±  0.208 %    88.335 ±  0.580 %
+   4      20.9749 ±    1.0406      -0.01108 ±    0.00618       0.04456 ±    0.00308     4.891 ±  0.299 %    89.565 ±  0.478 %
+   5      16.5848 ±    0.7193      -0.00917 ±    0.00518       0.03800 ±    0.00249     4.607 ±  0.262 %    90.811 ±  0.404 %
+   6      18.7215 ±    0.7472      -0.00954 ±    0.00509       0.04528 ±    0.00224     5.137 ±  0.252 %    89.834 ±  0.386 %
+   7      21.3836 ±    0.8000      -0.00008 ±    0.00490       0.04909 ±    0.00199     5.227 ±  0.226 %    89.345 ±  0.365 %
+   8      23.6569 ±    0.8287       0.00271 ±    0.00471       0.05155 ±    0.00178     5.248 ±  0.202 %    88.685 ±  0.350 %
+   9      20.7981 ±    0.6838       0.00422 ±    0.00433       0.04807 ±    0.00160     5.076 ±  0.187 %    89.443 ±  0.320 %
+  10      23.1873 ±    0.7314      -0.00204 ±    0.00410       0.05020 ±    0.00148     5.046 ±  0.173 %    88.993 ±  0.309 %
+  11      20.7451 ±    0.6213      -0.00079 ±    0.00383       0.04784 ±    0.00136     4.998 ±  0.161 %    89.469 ±  0.289 %
+  12      19.8598 ±    0.5643      -0.00567 ±    0.00364       0.04770 ±    0.00128     5.168 ±  0.155 %    89.492 ±  0.277 %
+  13      19.7849 ±    0.5377      -0.00211 ±    0.00350       0.04857 ±    0.00120     5.245 ±  0.146 %    89.240 ±  0.269 %
+  14      20.9128 ±    0.5502      -0.00396 ±    0.00342       0.04971 ±    0.00113     5.239 ±  0.137 %    88.975 ±  0.262 %
+  15      20.1519 ±    0.5098      -0.00425 ±    0.00332       0.04973 ±    0.00107     5.337 ±  0.131 %    89.078 ±  0.252 %
+  16      18.6664 ±    0.4549      -0.00443 ±    0.00315       0.04782 ±    0.00102     5.274 ±  0.126 %    89.479 ±  0.240 %
+  17      18.5783 ±    0.4353      -0.00469 ±    0.00302       0.04728 ±    0.00097     5.242 ±  0.120 %    89.397 ±  0.233 %
+  18      19.1103 ±    0.4339      -0.00485 ±    0.00290       0.04656 ±    0.00092     5.224 ±  0.116 %    89.394 ±  0.227 %
+  19      19.3085 ±    0.4256      -0.00258 ±    0.00282       0.04711 ±    0.00088     5.253 ±  0.111 %    89.299 ±  0.222 %
+  20      19.7024 ±    0.4220      -0.00387 ±    0.00276       0.04716 ±    0.00084     5.231 ±  0.107 %    89.262 ±  0.216 %
+  21      19.8591 ±    0.4144      -0.00351 ±    0.00268       0.04710 ±    0.00082     5.209 ±  0.103 %    89.308 ±  0.211 %
+  22      19.1254 ±    0.3875      -0.00317 ±    0.00257       0.04570 ±    0.00078     5.159 ±  0.101 %    89.576 ±  0.204 %
+  23      18.1151 ±    0.3572      -0.00377 ±    0.00248       0.04447 ±    0.00077     5.115 ±  0.100 %    89.885 ±  0.197 %
+  24      18.2710 ±    0.3538      -0.00296 ±    0.00244       0.04467 ±    0.00075     5.161 ±  0.098 %    89.781 ±  0.193 %
+  25      17.6049 ±    0.3328      -0.00279 ±    0.00236       0.04338 ±    0.00072     5.088 ±  0.095 %    89.975 ±  0.188 %
+  26      17.8942 ±    0.3316      -0.00291 ±    0.00231       0.04337 ±    0.00069     5.059 ±  0.092 %    89.943 ±  0.184 %
+  27      17.0782 ±    0.3097      -0.00165 ±    0.00229       0.04387 ±    0.00071     5.213 ±  0.095 %    90.000 ±  0.181 %
+  28      17.2875 ±    0.3074      -0.00173 ±    0.00224       0.04372 ±    0.00068     5.201 ±  0.093 %    89.994 ±  0.177 %
+  29      17.5791 ±    0.3081      -0.00238 ±    0.00222       0.04433 ±    0.00067     5.221 ±  0.090 %    89.878 ±  0.175 %
+  30      17.9585 ±    0.3092      -0.00347 ±    0.00219       0.04487 ±    0.00066     5.215 ±  0.088 %    89.713 ±  0.173 %
+  31      17.8034 ±    0.3002      -0.00278 ±    0.00215       0.04463 ±    0.00064     5.224 ±  0.087 %    89.749 ±  0.170 %
+  32      17.3903 ±    0.2882      -0.01190 ±    0.00246       0.05683 ±    0.00117     6.108 ±  0.117 %    89.260 ±  0.171 %
+  33      17.6195 ±    0.2873      -0.01280 ±    0.00241       0.05650 ±    0.00114     6.078 ±  0.115 %    89.188 ±  0.169 %
+  34      17.6545 ±    0.2828      -0.01312 ±    0.00236       0.05607 ±    0.00110     6.052 ±  0.112 %    89.098 ±  0.167 %
+  35      17.8969 ±    0.2831      -0.01257 ±    0.00233       0.05668 ��    0.00108     6.079 ±  0.110 %    88.996 ±  0.165 %
+  36      18.3681 ±    0.2879      -0.01284 ±    0.00233       0.05805 ±    0.00107     6.104 ±  0.107 %    88.816 ±  0.164 %
+  37      18.5831 ±    0.2872      -0.01327 ±    0.00228       0.05735 ±    0.00104     6.053 ±  0.105 %    88.888 ±  0.162 %
+  38      18.9709 ±    0.2897      -0.01409 ±    0.00224       0.05711 ±    0.00102     6.017 ±  0.103 %    88.797 ±  0.160 %
+  39      19.1935 ±    0.2891      -0.01424 ±    0.00220       0.05668 ±    0.00099     5.984 ±  0.101 %    88.794 ±  0.158 %
+  40      19.2943 ±    0.2866      -0.01487 ±    0.00216       0.05641 ±    0.00097     5.952 ±  0.099 %    88.761 ±  0.156 %
+  41      19.5044 ±    0.2863      -0.01604 ±    0.00213       0.05616 ±    0.00095     5.919 ±  0.097 %    88.713 ±  0.155 %
+  42      19.7541 ±    0.2869      -0.01621 ±    0.00210       0.05625 ±    0.00093     5.898 ±  0.096 %    88.658 ±  0.153 %
+  43      20.0280 ±    0.2875      -0.01727 ±    0.00207       0.05609 ±    0.00091     5.880 ±  0.094 %    88.631 ±  0.151 %
+  44      19.6851 ±    0.2785      -0.01705 ±    0.00203       0.05497 ±    0.00089     5.822 ±  0.093 %    88.785 ±  0.149 %
+
+====== Perplexity statistics ======
+Mean PPL(Q)                   :  19.685105 ±   0.278469
+Mean PPL(base)                :  20.023532 ±   0.285215
+Cor(ln(PPL(Q)), ln(PPL(base))):  98.98%
+Mean ln(PPL(Q)/PPL(base))     :  -0.017046 ±   0.002029
+Mean PPL(Q)/PPL(base)         :   0.983099 ±   0.001994
+Mean PPL(Q)-PPL(base)         :  -0.338427 ±   0.040788
+
+====== KL divergence statistics ======
+Mean    KLD:   0.054971 ±   0.000887
+Maximum KLD:  15.123309
+99.9%   KLD:   2.112881
+99.0%   KLD:   0.528482
+95.0%   KLD:   0.179696
+90.0%   KLD:   0.109617
+Median  KLD:   0.022553
+10.0%   KLD:   0.001180
+ 5.0%   KLD:   0.000204
+ 1.0%   KLD:   0.000005
+ 0.1%   KLD:  -0.000001
+Minimum KLD:  -0.000105
+
+====== Token probability statistics ======
+Mean    Δp: -0.061 ± 0.027 %
+Maximum Δp: 99.631%
+99.9%   Δp: 50.258%
+99.0%   Δp: 17.778%
+95.0%   Δp:  6.754%
+90.0%   Δp:  3.472%
+75.0%   Δp:  0.548%
+Median  Δp:  0.000%
+25.0%   Δp: -0.714%
+10.0%   Δp: -3.974%
+ 5.0%   Δp: -7.296%
+ 1.0%   Δp: -17.461%
+ 0.1%   Δp: -40.127%
+Minimum Δp: -91.966%
+RMS Δp    :  5.822 ± 0.093 %
+Same top p: 88.785 ± 0.149 %
+

reports/kld2048-Q6_K.txt

@@ -0,0 +1,91 @@
+0.00.753.072 I 
+0.00.753.091 I system_info: n_threads = 12 (n_threads_batch = 12) / 16 | MTL : EMBED_LIBRARY = 1 | CPU : NEON = 1 | ARM_FMA = 1 | FP16_VA = 1 | MATMUL_INT8 = 1 | DOTPROD = 1 | SME = 1 | ACCELERATE = 1 | OPENMP = 1 | REPACK = 1 | 
+0.00.766.732 I kl_divergence: computing over 44 chunks, n_ctx=2048, batch_size=2048, n_seq=1
+0.05.072.707 I kl_divergence: 4.31 seconds per pass - ETA 3.15 minutes
+
+chunk             PPL               ln(PPL(Q)/PPL(base))          KL Divergence              Δp RMS            Same top p
+   1      40.0041 ±    4.0847       0.02035 ±    0.01294       0.04754 ±    0.00217     5.017 ±  0.394 %    86.315 ±  1.075 %
+   2      43.1959 ±    3.1069      -0.00157 ±    0.00981       0.05240 ±    0.00169     5.029 ±  0.265 %    86.510 ±  0.755 %
+   3      30.7769 ±    1.8008      -0.00463 ±    0.00686       0.03930 ±    0.00120     4.418 ±  0.209 %    89.215 ±  0.560 %
+   4      21.0446 ±    1.0486      -0.00777 ±    0.00564       0.03561 ±    0.00234     4.555 ±  0.312 %    90.616 ±  0.456 %
+   5      16.6295 ±    0.7240      -0.00648 ±    0.00462       0.02974 ±    0.00188     4.205 ±  0.272 %    91.984 ±  0.380 %
+   6      18.8923 ±    0.7585      -0.00047 ±    0.00447       0.03427 ±    0.00164     4.595 ±  0.233 %    91.137 ±  0.363 %
+   7      21.4123 ±    0.8052       0.00126 ±    0.00436       0.03803 ±    0.00150     4.725 ±  0.210 %    90.476 ±  0.347 %
+   8      23.6670 ±    0.8334       0.00314 ±    0.00425       0.04068 ±    0.00135     4.725 ±  0.186 %    89.944 ±  0.332 %
+   9      20.7545 ±    0.6848       0.00213 ±    0.00387       0.03769 ±    0.00121     4.555 ±  0.173 %    90.518 ±  0.305 %
+  10      23.1620 ±    0.7331      -0.00314 ±    0.00368       0.03983 ±    0.00113     4.540 ±  0.159 %    89.873 ±  0.298 %
+  11      20.6827 ±    0.6208      -0.00380 ±    0.00342       0.03767 ±    0.00103     4.448 ±  0.148 %    90.483 ±  0.277 %
+  12      20.0255 ±    0.5712       0.00263 ±    0.00340       0.04175 ±    0.00102     4.825 ±  0.146 %    90.135 ±  0.269 %
+  13      19.9151 ±    0.5433       0.00445 ±    0.00324       0.04214 ±    0.00096     4.911 ±  0.138 %    90.044 ±  0.260 %
+  14      20.9747 ±    0.5539      -0.00100 ±    0.00320       0.04403 ±    0.00093     4.946 ±  0.129 %    89.548 ±  0.256 %
+  15      20.2122 ±    0.5131      -0.00127 ±    0.00310       0.04383 ±    0.00088     5.037 ±  0.123 %    89.606 ±  0.246 %
+  16      18.7312 ±    0.4580      -0.00096 ±    0.00293       0.04176 ±    0.00083     4.944 ±  0.118 %    89.968 ±  0.235 %
+  17      18.6427 ±    0.4385      -0.00122 ±    0.00282       0.04110 ±    0.00079     4.885 ±  0.113 %    89.960 ±  0.228 %
+  18      19.1412 ±    0.4364      -0.00323 ±    0.00270       0.04022 ±    0.00075     4.838 ±  0.108 %    90.089 ±  0.220 %
+  19      19.3194 ±    0.4275      -0.00201 ±    0.00263       0.04049 ±    0.00072     4.851 ±  0.104 %    89.993 ±  0.215 %
+  20      19.7105 ±    0.4240      -0.00346 ±    0.00256       0.04038 ±    0.00069     4.818 ±  0.100 %    89.922 ±  0.210 %
+  21      19.8347 ±    0.4156      -0.00474 ±    0.00247       0.03983 ±    0.00066     4.776 ±  0.097 %    90.029 ±  0.204 %
+  22      19.1081 ±    0.3888      -0.00407 ±    0.00237       0.03851 ±    0.00063     4.704 ±  0.094 %    90.287 ±  0.197 %
+  23      18.0961 ±    0.3583      -0.00482 ±    0.00229       0.03743 ±    0.00062     4.667 ±  0.094 %    90.501 ±  0.191 %
+  24      18.2480 ±    0.3546      -0.00422 ±    0.00226       0.03768 ±    0.00060     4.705 ±  0.091 %    90.376 ±  0.188 %
+  25      17.5764 ±    0.3333      -0.00442 ±    0.00218       0.03654 ±    0.00058     4.641 ±  0.089 %    90.588 ±  0.183 %
+  26      17.8519 ±    0.3317      -0.00528 ±    0.00213       0.03635 ±    0.00056     4.628 ±  0.087 %    90.567 ±  0.179 %
+  27      17.0149 ±    0.3091      -0.00536 ±    0.00210       0.03642 ±    0.00056     4.725 ±  0.089 %    90.681 ±  0.175 %
+  28      17.2258 ±    0.3069      -0.00531 ±    0.00205       0.03622 ±    0.00054     4.720 ±  0.086 %    90.661 ±  0.172 %
+  29      17.5403 ±    0.3081      -0.00458 ±    0.00203       0.03674 ±    0.00053     4.736 ±  0.084 %    90.559 ±  0.170 %
+  30      17.9357 ±    0.3095      -0.00474 ±    0.00201       0.03732 ±    0.00052     4.744 ±  0.081 %    90.443 ±  0.168 %
+  31      17.7863 ±    0.3006      -0.00374 ±    0.00197       0.03708 ±    0.00051     4.759 ±  0.081 %    90.483 ±  0.165 %
+  32      17.5273 ±    0.2914      -0.00405 ±    0.00193       0.03644 ±    0.00050     4.739 ±  0.079 %    90.619 ±  0.161 %
+  33      17.7630 ±    0.2905      -0.00468 ±    0.00190       0.03652 ±    0.00049     4.728 ±  0.077 %    90.521 ±  0.159 %
+  34      17.8153 ±    0.2863      -0.00406 ±    0.00186       0.03645 ±    0.00047     4.726 ±  0.076 %    90.417 ±  0.158 %
+  35      18.0621 ±    0.2868      -0.00338 ±    0.00185       0.03706 ��    0.00048     4.761 ±  0.075 %    90.353 ±  0.156 %
+  36      18.5182 ±    0.2913      -0.00470 ±    0.00264       0.07751 ±    0.00197     6.147 ±  0.122 %    89.109 ±  0.162 %
+  37      18.7423 ±    0.2907      -0.00474 ±    0.00258       0.07612 ±    0.00192     6.090 ±  0.120 %    89.205 ±  0.160 %
+  38      19.1342 ±    0.2933      -0.00552 ±    0.00253       0.07527 ±    0.00187     6.057 ±  0.118 %    89.126 ±  0.158 %
+  39      19.3702 ±    0.2930      -0.00508 ±    0.00248       0.07438 ±    0.00182     6.019 ±  0.116 %    89.109 ±  0.156 %
+  40      19.4712 ±    0.2904      -0.00574 ±    0.00243       0.07353 ±    0.00178     5.983 ±  0.114 %    89.074 ±  0.154 %
+  41      19.7055 ±    0.2906      -0.00579 ±    0.00239       0.07265 ±    0.00173     5.945 ±  0.112 %    89.071 ±  0.152 %
+  42      19.9599 ±    0.2912      -0.00584 ±    0.00235       0.07225 ±    0.00169     5.933 ±  0.109 %    89.010 ±  0.151 %
+  43      20.2370 ±    0.2918      -0.00689 ±    0.00231       0.07163 ±    0.00166     5.902 ±  0.108 %    88.997 ±  0.149 %
+  44      19.8878 ±    0.2826      -0.00680 ±    0.00226       0.07013 ±    0.00162     5.840 ±  0.106 %    89.161 ±  0.147 %
+
+====== Perplexity statistics ======
+Mean PPL(Q)                   :  19.887821 ±   0.282623
+Mean PPL(base)                :  20.023532 ±   0.285215
+Cor(ln(PPL(Q)), ln(PPL(base))):  98.74%
+Mean ln(PPL(Q)/PPL(base))     :  -0.006801 ±   0.002259
+Mean PPL(Q)/PPL(base)         :   0.993222 ±   0.002243
+Mean PPL(Q)-PPL(base)         :  -0.135711 ±   0.045139
+
+====== KL divergence statistics ======
+Mean    KLD:   0.070127 ±   0.001619
+Maximum KLD:  13.524951
+99.9%   KLD:   5.157535
+99.0%   KLD:   1.238282
+95.0%   KLD:   0.171232
+90.0%   KLD:   0.097467
+Median  KLD:   0.017696
+10.0%   KLD:   0.000818
+ 5.0%   KLD:   0.000128
+ 1.0%   KLD:   0.000003
+ 0.1%   KLD:  -0.000002
+Minimum KLD:  -0.000086
+
+====== Token probability statistics ======
+Mean    Δp: -0.018 ± 0.028 %
+Maximum Δp: 99.446%
+99.9%   Δp: 53.868%
+99.0%   Δp: 16.426%
+95.0%   Δp:  6.337%
+90.0%   Δp:  3.328%
+75.0%   Δp:  0.540%
+Median  Δp:  0.000%
+25.0%   Δp: -0.562%
+10.0%   Δp: -3.485%
+ 5.0%   Δp: -6.557%
+ 1.0%   Δp: -17.290%
+ 0.1%   Δp: -49.032%
+Minimum Δp: -99.623%
+RMS Δp    :  5.840 ± 0.106 %
+Same top p: 89.161 ± 0.147 %
+

reports/kld2048-Q8_0.txt

@@ -0,0 +1,91 @@
+0.04.058.683 I 
+0.04.058.704 I system_info: n_threads = 12 (n_threads_batch = 12) / 16 | MTL : EMBED_LIBRARY = 1 | CPU : NEON = 1 | ARM_FMA = 1 | FP16_VA = 1 | MATMUL_INT8 = 1 | DOTPROD = 1 | SME = 1 | ACCELERATE = 1 | OPENMP = 1 | REPACK = 1 | 
+0.04.074.455 I kl_divergence: computing over 44 chunks, n_ctx=2048, batch_size=2048, n_seq=1
+0.07.642.683 I kl_divergence: 3.57 seconds per pass - ETA 2.62 minutes
+
+chunk             PPL               ln(PPL(Q)/PPL(base))          KL Divergence              Δp RMS            Same top p
+   1      39.3179 ±    3.9687       0.00304 ±    0.00447       0.00512 ±    0.00020     1.719 ±  0.116 %    95.992 ±  0.614 %
+   2      43.7425 ±    3.1358       0.01101 ±    0.00482       0.01071 ±    0.00056     2.402 ±  0.135 %    94.379 ±  0.509 %
+   3      31.1566 ±    1.8236       0.00764 ±    0.00333       0.00775 ±    0.00040     2.043 ±  0.108 %    95.797 ±  0.362 %
+   4      21.3343 ±    1.0652       0.00590 ±    0.00256       0.00627 ±    0.00030     1.885 ±  0.090 %    96.359 ±  0.293 %
+   5      16.8234 ±    0.7337       0.00512 ±    0.00210       0.00519 ±    0.00025     1.726 ±  0.079 %    96.950 ±  0.240 %
+   6      19.0176 ±    0.7650       0.00615 ±    0.00192       0.00556 ±    0.00022     1.787 ±  0.071 %    96.693 ±  0.228 %
+   7      21.5324 ±    0.8110       0.00686 ±    0.00177       0.00567 ±    0.00019     1.783 ±  0.064 %    96.411 ±  0.220 %
+   8      23.7777 ±    0.8379       0.00781 ±    0.00163       0.00568 ±    0.00017     1.738 ±  0.058 %    96.261 ±  0.210 %
+   9      20.8430 ±    0.6883       0.00638 ±    0.00148       0.00523 ±    0.00015     1.671 ±  0.054 %    96.470 ±  0.192 %
+  10      23.4044 ±    0.7421       0.00727 ±    0.00140       0.00541 ±    0.00016     1.658 ±  0.050 %    96.325 ±  0.186 %
+  11      20.9095 ±    0.6296       0.00710 ±    0.00132       0.00510 ±    0.00015     1.624 ±  0.047 %    96.561 ±  0.172 %
+  12      20.1019 ±    0.5738       0.00644 ±    0.00124       0.00504 ±    0.00014     1.643 ±  0.045 %    96.562 ±  0.164 %
+  13      19.9460 ±    0.5443       0.00600 ±    0.00118       0.00507 ±    0.00013     1.675 ±  0.042 %    96.534 ±  0.159 %
+  14      21.1268 ±    0.5585       0.00622 ±    0.00116       0.00532 ±    0.00013     1.688 ±  0.040 %    96.327 ±  0.157 %
+  15      20.3644 ±    0.5177       0.00623 ±    0.00111       0.00519 ±    0.00012     1.697 ±  0.038 %    96.357 ±  0.151 %
+  16      18.8566 ±    0.4618       0.00571 ±    0.00105       0.00494 ±    0.00011     1.667 ±  0.037 %    96.518 ±  0.143 %
+  17      18.7770 ±    0.4422       0.00596 ±    0.00101       0.00486 ±    0.00011     1.653 ±  0.035 %    96.527 ±  0.139 %
+  18      19.3249 ±    0.4413       0.00632 ±    0.00096       0.00475 ±    0.00010     1.634 ±  0.034 %    96.573 ±  0.134 %
+  19      19.4971 ±    0.4323       0.00714 ±    0.00093       0.00471 ±    0.00010     1.629 ±  0.032 %    96.584 ±  0.130 %
+  20      19.9216 ±    0.4295       0.00719 ±    0.00090       0.00470 ±    0.00009     1.623 ±  0.031 %    96.608 ±  0.127 %
+  21      20.0633 ±    0.4215       0.00672 ±    0.00087       0.00462 ±    0.00009     1.614 ±  0.031 %    96.639 ±  0.123 %
+  22      19.3105 ±    0.3940       0.00646 ±    0.00084       0.00448 ±    0.00008     1.589 ±  0.030 %    96.716 ±  0.119 %
+  23      18.2955 ±    0.3633       0.00614 ±    0.00080       0.00433 ±    0.00008     1.565 ±  0.029 %    96.791 ±  0.115 %
+  24      18.4489 ±    0.3596       0.00673 ±    0.00081       0.00434 ±    0.00008     1.577 ±  0.028 %    96.790 ±  0.112 %
+  25      17.7665 ±    0.3379       0.00634 ±    0.00078       0.00421 ±    0.00007     1.556 ±  0.028 %    96.876 ±  0.109 %
+  26      18.0634 ±    0.3369       0.00650 ±    0.00076       0.00423 ±    0.00007     1.570 ±  0.027 %    96.857 ±  0.107 %
+  27      17.2203 ±    0.3140       0.00664 ±    0.00076       0.00431 ±    0.00008     1.617 ±  0.031 %    96.854 ±  0.105 %
+  28      17.4329 ±    0.3118       0.00664 ±    0.00074       0.00427 ±    0.00007     1.609 ±  0.030 %    96.837 ±  0.103 %
+  29      17.7293 ±    0.3125       0.00613 ±    0.00074       0.00435 ±    0.00007     1.622 ±  0.029 %    96.794 ±  0.102 %
+  30      18.1300 ±    0.3139       0.00604 ±    0.00073       0.00440 ±    0.00007     1.617 ±  0.028 %    96.774 ±  0.101 %
+  31      17.9560 ±    0.3043       0.00576 ±    0.00071       0.00436 ±    0.00007     1.617 ±  0.028 %    96.806 ±  0.099 %
+  32      17.7479 ±    0.2960       0.00845 ±    0.00077       0.00503 ±    0.00009     1.865 ±  0.047 %    96.722 ±  0.098 %
+  33      18.0002 ±    0.2953       0.00858 ±    0.00075       0.00500 ±    0.00009     1.857 ±  0.046 %    96.673 ±  0.098 %
+  34      18.0432 ±    0.2910       0.00865 ±    0.00074       0.00496 ±    0.00008     1.848 ±  0.045 %    96.651 ±  0.096 %
+  35      18.2803 ±    0.2911       0.00863 ±    0.00073       0.00500 ��    0.00008     1.848 ±  0.044 %    96.612 ±  0.096 %
+  36      18.7684 ±    0.2961       0.00872 ±    0.00079       0.00611 ±    0.00011     1.952 ±  0.044 %    96.416 ±  0.097 %
+  37      18.9962 ±    0.2956       0.00872 ±    0.00078       0.00603 ±    0.00011     1.934 ±  0.043 %    96.418 ±  0.096 %
+  38      19.4098 ±    0.2985       0.00878 ±    0.00076       0.00599 ±    0.00011     1.922 ±  0.042 %    96.393 ±  0.095 %
+  39      19.6427 ±    0.2980       0.00889 ±    0.00075       0.00593 ±    0.00011     1.911 ±  0.042 %    96.393 ±  0.093 %
+  40      19.7654 ±    0.2958       0.00925 ±    0.00073       0.00590 ±    0.00010     1.904 ±  0.041 %    96.378 ±  0.092 %
+  41      20.0024 ±    0.2960       0.00917 ±    0.00072       0.00587 ±    0.00010     1.894 ±  0.040 %    96.366 ±  0.091 %
+  42      20.2606 ±    0.2967       0.00911 ±    0.00071       0.00585 ±    0.00010     1.887 ±  0.039 %    96.348 ±  0.090 %
+  43      20.5612 ±    0.2977       0.00900 ±    0.00070       0.00583 ±    0.00010     1.879 ±  0.038 %    96.329 ±  0.090 %
+  44      20.2006 ±    0.2882       0.00880 ±    0.00069       0.00571 ±    0.00009     1.860 ±  0.038 %    96.381 ±  0.088 %
+
+====== Perplexity statistics ======
+Mean PPL(Q)                   :  20.200587 ±   0.288209
+Mean PPL(base)                :  20.023532 ±   0.285215
+Cor(ln(PPL(Q)), ln(PPL(base))):  99.88%
+Mean ln(PPL(Q)/PPL(base))     :   0.008803 ±   0.000686
+Mean PPL(Q)/PPL(base)         :   1.008842 ±   0.000692
+Mean PPL(Q)-PPL(base)         :   0.177055 ±   0.014106
+
+====== KL divergence statistics ======
+Mean    KLD:   0.005708 ±   0.000095
+Maximum KLD:   1.475488
+99.9%   KLD:   0.233686
+99.0%   KLD:   0.060422
+95.0%   KLD:   0.019779
+90.0%   KLD:   0.011459
+Median  KLD:   0.002024
+10.0%   KLD:   0.000093
+ 5.0%   KLD:   0.000014
+ 1.0%   KLD:  -0.000000
+ 0.1%   KLD:  -0.000027
+Minimum KLD:  -0.000139
+
+====== Token probability statistics ======
+Mean    Δp: -0.129 ± 0.009 %
+Maximum Δp: 26.548%
+99.9%   Δp: 12.262%
+99.0%   Δp:  4.915%
+95.0%   Δp:  1.852%
+90.0%   Δp:  0.953%
+75.0%   Δp:  0.123%
+Median  Δp: -0.001%
+25.0%   Δp: -0.252%
+10.0%   Δp: -1.292%
+ 5.0%   Δp: -2.469%
+ 1.0%   Δp: -6.287%
+ 0.1%   Δp: -15.758%
+Minimum Δp: -56.803%
+RMS Δp    :  1.860 ± 0.038 %
+Same top p: 96.381 ± 0.088 %
+

reports/kld2048b-IQ2_M.txt

@@ -0,0 +1,91 @@
+0.01.613.406 I 
+0.01.613.424 I system_info: n_threads = 12 (n_threads_batch = 12) / 16 | MTL : EMBED_LIBRARY = 1 | CPU : NEON = 1 | ARM_FMA = 1 | FP16_VA = 1 | MATMUL_INT8 = 1 | DOTPROD = 1 | SME = 1 | ACCELERATE = 1 | OPENMP = 1 | REPACK = 1 | 
+0.01.629.925 I kl_divergence: computing over 44 chunks, n_ctx=2048, batch_size=2048, n_seq=1
+0.05.557.685 I kl_divergence: 3.93 seconds per pass - ETA 2.87 minutes
+
+chunk             PPL               ln(PPL(Q)/PPL(base))          KL Divergence              Δp RMS            Same top p
+   1      58.2738 ±    5.8636       0.39652 ±    0.04440       0.73148 ±    0.02548    17.754 ±  0.809 %    53.568 ±  1.560 %
+   2      28.1736 ±    1.9403      -0.42893 ±    0.04739       1.62751 ±    0.05135    27.095 ±  0.742 %    48.485 ±  1.105 %
+   3      24.1057 ±    1.3464      -0.24894 ±    0.03412       1.23011 ±    0.03642    23.821 ±  0.597 %    55.979 ±  0.896 %
+   4      18.1613 ±    0.8667      -0.15512 ±    0.02725       1.03879 ±    0.02884    22.542 ±  0.513 %    60.606 ±  0.764 %
+   5      15.2523 ±    0.6348      -0.09293 ±    0.02296       0.90438 ±    0.02390    21.904 ±  0.458 %    63.930 ±  0.672 %
+   6      18.4109 ±    0.7142      -0.02628 ±    0.02074       0.88345 ±    0.02052    21.337 ±  0.409 %    62.887 ±  0.617 %
+   7      22.3974 ±    0.8255       0.04625 ±    0.01934       0.86748 ±    0.01795    21.006 ±  0.371 %    61.346 ±  0.575 %
+   8      20.6588 ±    0.7044      -0.13280 ±    0.02011       1.13657 ±    0.02250    23.875 ±  0.373 %    58.798 ±  0.544 %
+   9      18.6723 ±    0.5957      -0.10359 ±    0.01844       1.06388 ±    0.02038    23.551 ±  0.352 %    60.302 ±  0.510 %
+  10      21.9508 ±    0.6830      -0.05685 ±    0.01731       1.03128 ±    0.01850    22.928 ±  0.331 %    59.638 ±  0.485 %
+  11      20.0201 ±    0.5899      -0.03637 ±    0.01640       0.99277 ±    0.01720    22.831 ±  0.314 %    60.739 ±  0.460 %
+  12      18.6809 ±    0.5225      -0.06687 ±    0.01563       1.01016 ±    0.01638    23.237 ±  0.298 %    60.834 ±  0.441 %
+  13      19.0078 ±    0.5098      -0.04218 ±    0.01478       0.97615 ±    0.01523    22.893 ±  0.283 %    60.944 ±  0.423 %
+  14      20.4682 ±    0.5313      -0.02545 ±    0.01420       0.96361 ±    0.01434    22.565 ±  0.271 %    60.529 ±  0.408 %
+  15      18.4287 ±    0.4568      -0.09364 ±    0.01414       1.03946 ±    0.01485    23.800 ±  0.265 %    60.169 ±  0.395 %
+  16      17.4307 ±    0.4156      -0.07292 ±    0.01355       1.00682 ±    0.01413    23.706 ±  0.257 %    60.997 ±  0.381 %
+  17      17.6917 ±    0.4072      -0.05359 ±    0.01300       0.98756 ±    0.01338    23.431 ±  0.248 %    60.882 ±  0.370 %
+  18      16.8473 ±    0.3737      -0.13088 ±    0.01310       1.06775 ±    0.01353    24.646 ±  0.244 %    60.041 ±  0.361 %
+  19      17.4819 ±    0.3780      -0.10195 ±    0.01263       1.04620 ±    0.01290    24.367 ±  0.236 %    59.958 ±  0.351 %
+  20      18.3668 ±    0.3873      -0.07407 ±    0.01226       1.03257 ±    0.01232    24.077 ±  0.229 %    59.780 ±  0.343 %
+  21      18.8620 ±    0.3886      -0.05502 ±    0.01190       1.01193 ±    0.01180    23.846 ±  0.222 %    59.791 ±  0.335 %
+  22      18.3624 ±    0.3674      -0.04388 ±    0.01146       0.98364 ±    0.01133    23.573 ±  0.216 %    60.375 ±  0.326 %
+  23      17.5539 ±    0.3417      -0.03524 ±    0.01107       0.95691 ±    0.01090    23.362 ±  0.211 %    61.001 ±  0.318 %
+  24      18.0106 ±    0.3447      -0.01731 ±    0.01082       0.94846 ±    0.01053    23.198 ±  0.206 %    60.952 ±  0.311 %
+  25      17.4314 ±    0.3253      -0.01270 ±    0.01045       0.92335 ±    0.01015    22.984 ±  0.201 %    61.529 ±  0.304 %
+  26      17.9211 ±    0.3279      -0.00141 ±    0.01018       0.90906 ±    0.00979    22.789 ±  0.196 %    61.456 ±  0.298 %
+  27      17.2064 ±    0.3074       0.00583 ±    0.00992       0.89507 ±    0.00951    22.838 ±  0.192 %    61.859 ±  0.292 %
+  28      17.6901 ±    0.3111       0.02129 ±    0.00969       0.88479 ±    0.00921    22.665 ±  0.188 %    61.877 ±  0.287 %
+  29      18.2022 ±    0.3160       0.03246 ±    0.00952       0.88060 ±    0.00895    22.602 ±  0.184 %    61.691 ±  0.282 %
+  30      18.8616 ±    0.3227       0.04560 ±    0.00936       0.87818 ±    0.00871    22.473 ±  0.180 %    61.492 ±  0.278 %
+  31      18.7717 ±    0.3146       0.05019 ±    0.00914       0.86753 ±    0.00846    22.370 ±  0.177 %    61.590 ±  0.273 %
+  32      18.8487 ±    0.3107       0.06863 ±    0.00907       0.87097 ±    0.00828    22.641 ±  0.175 %    61.431 ±  0.269 %
+  33      19.2552 ±    0.3123       0.07598 ±    0.00889       0.86278 ±    0.00806    22.499 ±  0.172 %    61.358 ±  0.265 %
+  34      19.5123 ±    0.3115       0.08693 ±    0.00871       0.85368 ±    0.00785    22.374 ±  0.168 %    61.287 ±  0.261 %
+  35      19.9289 ±    0.3141       0.09497 ±    0.00857       0.84976 ��    0.00765    22.335 ±  0.165 %    61.170 ±  0.258 %
+  36      19.7929 ±    0.3065       0.06187 ±    0.00864       0.88110 ±    0.00775    22.485 ±  0.163 %    60.549 ±  0.255 %
+  37      20.0909 ±    0.3068       0.06474 ±    0.00846       0.87049 ±    0.00756    22.319 ±  0.160 %    60.617 ±  0.251 %
+  38      20.6278 ±    0.3115       0.06964 ±    0.00831       0.86380 ±    0.00739    22.192 ±  0.158 %    60.441 ±  0.248 %
+  39      21.0228 ±    0.3135       0.07679 ±    0.00817       0.85589 ±    0.00721    22.061 ±  0.155 %    60.431 ±  0.245 %
+  40      21.2105 ±    0.3123       0.07982 ±    0.00803       0.84951 ±    0.00706    21.953 ±  0.153 %    60.411 ±  0.242 %
+  41      21.5474 ±    0.3138       0.08357 ±    0.00790       0.84404 ±    0.00691    21.850 ±  0.151 %    60.401 ±  0.239 %
+  42      21.9250 ±    0.3162       0.08806 ±    0.00779       0.83956 ±    0.00677    21.753 ±  0.148 %    60.364 ±  0.236 %
+  43      22.3737 ±    0.3191       0.09349 ±    0.00767       0.83571 ±    0.00664    21.655 ±  0.146 %    60.299 ±  0.233 %
+  44      21.9919 ±    0.3091       0.09377 ±    0.00751       0.82111 ±    0.00651    21.470 ±  0.145 %    60.690 ±  0.230 %
+
+====== Perplexity statistics ======
+Mean PPL(Q)                   :  21.991922 ±   0.309087
+Mean PPL(base)                :  20.023532 ±   0.285215
+Cor(ln(PPL(Q)), ln(PPL(base))):  85.91%
+Mean ln(PPL(Q)/PPL(base))     :   0.093767 ±   0.007514
+Mean PPL(Q)/PPL(base)         :   1.098304 ±   0.008253
+Mean PPL(Q)-PPL(base)         :   1.968390 ±   0.159436
+
+====== KL divergence statistics ======
+Mean    KLD:   0.821109 ±   0.006506
+Maximum KLD:  27.317856
+99.9%   KLD:  13.975519
+99.0%   KLD:   7.225517
+95.0%   KLD:   2.905154
+90.0%   KLD:   1.848341
+Median  KLD:   0.419832
+10.0%   KLD:   0.037137
+ 5.0%   KLD:   0.008512
+ 1.0%   KLD:   0.000367
+ 0.1%   KLD:   0.000015
+Minimum KLD:  -0.000003
+
+====== Token probability statistics ======
+Mean    Δp: -2.527 ± 0.100 %
+Maximum Δp: 99.984%
+99.9%   Δp: 99.142%
+99.0%   Δp: 77.508%
+95.0%   Δp: 27.281%
+90.0%   Δp: 12.375%
+75.0%   Δp:  1.135%
+Median  Δp: -0.116%
+25.0%   Δp: -6.481%
+10.0%   Δp: -23.965%
+ 5.0%   Δp: -38.637%
+ 1.0%   Δp: -72.888%
+ 0.1%   Δp: -96.885%
+Minimum Δp: -99.975%
+RMS Δp    : 21.470 ± 0.145 %
+Same top p: 60.690 ± 0.230 %
+

reports/kld2048b-IQ4_XS.txt

@@ -0,0 +1,91 @@
+0.02.378.266 I 
+0.02.378.288 I system_info: n_threads = 12 (n_threads_batch = 12) / 16 | MTL : EMBED_LIBRARY = 1 | CPU : NEON = 1 | ARM_FMA = 1 | FP16_VA = 1 | MATMUL_INT8 = 1 | DOTPROD = 1 | SME = 1 | ACCELERATE = 1 | OPENMP = 1 | REPACK = 1 | 
+0.02.392.915 I kl_divergence: computing over 44 chunks, n_ctx=2048, batch_size=2048, n_seq=1
+0.05.866.452 I kl_divergence: 3.47 seconds per pass - ETA 2.53 minutes
+
+chunk             PPL               ln(PPL(Q)/PPL(base))          KL Divergence              Δp RMS            Same top p
+   1      40.8014 ±    4.0883       0.04008 ±    0.02030       0.13138 ±    0.00527     7.765 ±  0.478 %    76.637 ±  1.324 %
+   2      46.1092 ±    3.3048       0.06370 ±    0.01516       0.13641 ±    0.00396     8.106 ±  0.383 %    77.713 ±  0.920 %
+   3      32.8071 ±    1.9299       0.05926 ±    0.01117       0.11340 ±    0.00316     7.545 ±  0.350 %    81.069 ±  0.707 %
+   4      22.4515 ±    1.1303       0.05694 ±    0.00948       0.10398 ±    0.00281     7.428 ±  0.296 %    83.431 ±  0.581 %
+   5      17.6272 ±    0.7763       0.05179 ±    0.00817       0.09287 ±    0.00269     7.189 ±  0.283 %    85.415 ±  0.494 %
+   6      19.9152 ±    0.8074       0.05226 ±    0.00775       0.10057 ±    0.00249     7.343 ±  0.245 %    84.572 ±  0.461 %
+   7      22.5669 ±    0.8587       0.05378 ±    0.00731       0.10573 ±    0.00230     7.330 ±  0.220 %    83.675 ±  0.437 %
+   8      24.8553 ±    0.8845       0.05213 ±    0.00703       0.11585 ±    0.00230     7.539 ±  0.207 %    82.564 ±  0.419 %
+   9      21.6370 ±    0.7205       0.04377 ±    0.00651       0.11104 ±    0.00220     7.532 ±  0.195 %    83.263 ±  0.389 %
+  10      24.0920 ±    0.7686       0.03623 ±    0.00625       0.11825 ±    0.00211     7.547 ±  0.183 %    82.444 ±  0.376 %
+  11      21.4854 ±    0.6509       0.03427 ±    0.00593       0.11635 ±    0.00203     7.631 ±  0.176 %    82.982 ±  0.354 %
+  12      20.5664 ±    0.5904       0.02929 ±    0.00562       0.11540 ±    0.00191     7.732 ±  0.165 %    83.170 ±  0.338 %
+  13      20.2696 ±    0.5561       0.02209 ±    0.00541       0.11621 ±    0.00181     7.861 ±  0.156 %    82.886 ±  0.327 %
+  14      21.4866 ±    0.5707       0.02311 ±    0.00524       0.11889 ±    0.00175     7.874 ±  0.150 %    82.642 ±  0.316 %
+  15      19.1890 ±    0.4870      -0.05322 ±    0.00702       0.25911 ±    0.00767    12.351 ±  0.258 %    80.828 ±  0.318 %
+  16      17.8779 ±    0.4370      -0.04759 ±    0.00666       0.24714 ±    0.00722    12.113 ±  0.249 %    81.433 ±  0.304 %
+  17      17.9925 ±    0.4230      -0.03672 ±    0.00639       0.24029 ±    0.00681    11.982 ±  0.238 %    81.347 ±  0.295 %
+  18      18.6280 ±    0.4250      -0.03041 ±    0.00610       0.23248 ±    0.00644    11.771 ±  0.229 %    81.340 ±  0.287 %
+  19      18.9205 ±    0.4190      -0.02287 ±    0.00588       0.22702 ±    0.00610    11.628 ±  0.220 %    81.309 ±  0.280 %
+  20      19.4109 ±    0.4180      -0.01878 ±    0.00568       0.22270 ±    0.00581    11.457 ±  0.213 %    81.251 ±  0.273 %
+  21      19.6166 ±    0.4115      -0.01580 ±    0.00546       0.21691 ±    0.00554    11.292 ±  0.206 %    81.409 ±  0.265 %
+  22      18.9276 ±    0.3856      -0.01356 ±    0.00525       0.20974 ±    0.00529    11.132 ±  0.200 %    81.778 ±  0.257 %
+  23      17.9744 ±    0.3565      -0.01156 ±    0.00506       0.20318 ±    0.00507    11.028 ±  0.195 %    82.150 ±  0.250 %
+  24      18.2363 ±    0.3555      -0.00486 ±    0.00496       0.20124 ±    0.00487    11.007 ±  0.189 %    82.030 ±  0.245 %
+  25      17.5697 ±    0.3343      -0.00480 ±    0.00478       0.19488 ±    0.00468    10.837 ±  0.185 %    82.389 ±  0.238 %
+  26      17.9133 ±    0.3342      -0.00184 ±    0.00465       0.19186 ±    0.00451    10.751 ±  0.180 %    82.356 ±  0.234 %
+  27      17.1145 ±    0.3124       0.00048 ±    0.00457       0.19081 ±    0.00438    10.812 ±  0.175 %    82.470 ±  0.229 %
+  28      17.3774 ±    0.3113       0.00346 ±    0.00445       0.18779 ±    0.00423    10.712 ±  0.171 %    82.495 ±  0.225 %
+  29      17.7102 ±    0.3128       0.00505 ±    0.00437       0.18741 ±    0.00410    10.681 ±  0.166 %    82.354 ±  0.221 %
+  30      18.1273 ±    0.3146       0.00589 ±    0.00429       0.18601 ±    0.00396    10.586 ±  0.163 %    82.255 ±  0.218 %
+  31      18.0144 ±    0.3063       0.00901 ±    0.00418       0.18324 ±    0.00384    10.512 ±  0.159 %    82.304 ±  0.214 %
+  32      18.3101 ±    0.3064       0.03964 ±    0.00475       0.22681 ±    0.00426    12.223 ±  0.173 %    81.128 ±  0.216 %
+  33      18.5720 ±    0.3056       0.03985 ±    0.00464       0.22342 ±    0.00414    12.120 ±  0.170 %    81.078 ±  0.213 %
+  34      18.6240 ±    0.3012       0.04034 ±    0.00454       0.22017 ±    0.00402    12.009 ±  0.166 %    81.059 ±  0.210 %
+  35      18.9015 ±    0.3020       0.04204 ±    0.00445       0.21834 ��    0.00391    11.930 ±  0.163 %    81.045 ±  0.207 %
+  36      19.3547 ±    0.3059       0.03948 ±    0.00464       0.24281 ±    0.00405    12.260 ±  0.160 %    80.184 ±  0.208 %
+  37      19.5866 ±    0.3052       0.03932 ±    0.00453       0.23854 ±    0.00395    12.143 ±  0.158 %    80.312 ±  0.204 %
+  38      20.0163 ±    0.3081       0.03955 ±    0.00443       0.23544 ±    0.00385    12.042 ±  0.155 %    80.257 ±  0.202 %
+  39      20.2668 ±    0.3077       0.04017 ±    0.00434       0.23233 ±    0.00375    11.937 ±  0.152 %    80.264 ±  0.199 %
+  40      20.3846 ±    0.3054       0.04010 ±    0.00425       0.22910 ±    0.00366    11.841 ±  0.150 %    80.347 ±  0.196 %
+  41      20.6113 ±    0.3052       0.03915 ±    0.00417       0.22629 ±    0.00357    11.752 ±  0.148 %    80.335 ±  0.194 %
+  42      20.8683 ±    0.3056       0.03866 ±    0.00410       0.22436 ±    0.00349    11.685 ±  0.145 %    80.296 ±  0.192 %
+  43      21.1594 ±    0.3063       0.03768 ±    0.00403       0.22184 ±    0.00341    11.595 ±  0.143 %    80.309 ±  0.190 %
+  44      20.7806 ±    0.2965       0.03711 ±    0.00395       0.21727 ±    0.00334    11.474 ±  0.141 %    80.554 ±  0.187 %
+
+====== Perplexity statistics ======
+Mean PPL(Q)                   :  20.780607 ±   0.296500
+Mean PPL(base)                :  20.023532 ±   0.285215
+Cor(ln(PPL(Q)), ln(PPL(base))):  96.17%
+Mean ln(PPL(Q)/PPL(base))     :   0.037112 ±   0.003946
+Mean PPL(Q)/PPL(base)         :   1.037809 ±   0.004096
+Mean PPL(Q)-PPL(base)         :   0.757076 ±   0.081286
+
+====== KL divergence statistics ======
+Mean    KLD:   0.217275 ±   0.003340
+Maximum KLD:  17.964737
+99.9%   KLD:   9.634006
+99.0%   KLD:   3.159044
+95.0%   KLD:   0.760046
+90.0%   KLD:   0.369870
+Median  KLD:   0.066936
+10.0%   KLD:   0.003958
+ 5.0%   KLD:   0.000743
+ 1.0%   KLD:   0.000018
+ 0.1%   KLD:   0.000000
+Minimum KLD:  -0.000052
+
+====== Token probability statistics ======
+Mean    Δp: -0.564 ± 0.054 %
+Maximum Δp: 99.982%
+99.9%   Δp: 91.608%
+99.0%   Δp: 34.479%
+95.0%   Δp: 11.713%
+90.0%   Δp:  5.915%
+75.0%   Δp:  0.778%
+Median  Δp: -0.004%
+25.0%   Δp: -1.506%
+10.0%   Δp: -7.762%
+ 5.0%   Δp: -14.633%
+ 1.0%   Δp: -39.807%
+ 0.1%   Δp: -93.063%
+Minimum Δp: -99.936%
+RMS Δp    : 11.474 ± 0.141 %
+Same top p: 80.554 ± 0.187 %
+

reports/kld2048b-Q3_K_M.txt

@@ -0,0 +1,91 @@
+0.02.201.588 I 
+0.02.201.608 I system_info: n_threads = 12 (n_threads_batch = 12) / 16 | MTL : EMBED_LIBRARY = 1 | CPU : NEON = 1 | ARM_FMA = 1 | FP16_VA = 1 | MATMUL_INT8 = 1 | DOTPROD = 1 | SME = 1 | ACCELERATE = 1 | OPENMP = 1 | REPACK = 1 | 
+0.02.216.355 I kl_divergence: computing over 44 chunks, n_ctx=2048, batch_size=2048, n_seq=1
+0.06.094.651 I kl_divergence: 3.88 seconds per pass - ETA 2.83 minutes
+
+chunk             PPL               ln(PPL(Q)/PPL(base))          KL Divergence              Δp RMS            Same top p
+   1      44.0050 ±    4.3037       0.11567 ±    0.02992       0.28583 ±    0.01011    11.510 ±  0.624 %    68.817 ±  1.449 %
+   2      49.0206 ±    3.5001       0.12493 ±    0.02173       0.28682 ±    0.00748    11.258 ±  0.444 %    70.430 ±  1.009 %
+   3      34.3613 ±    2.0141       0.10554 ±    0.01647       0.24723 ±    0.00612    10.763 ±  0.392 %    74.356 ±  0.788 %
+   4      23.0288 ±    1.1546       0.08233 ±    0.01416       0.23331 ±    0.00578    11.082 ±  0.354 %    76.808 ±  0.660 %
+   5      18.0180 ±    0.7893       0.07372 ±    0.01208       0.20844 ±    0.00508    11.076 ±  0.339 %    79.179 ±  0.568 %
+   6      20.7834 ±    0.8412       0.09494 ±    0.01133       0.23622 ±    0.00501    11.508 ±  0.308 %    77.452 ±  0.533 %
+   7      23.7445 ±    0.8998       0.10465 ±    0.01061       0.24902 ±    0.00473    11.596 ±  0.281 %    76.316 ±  0.502 %
+   8      26.1189 ±    0.9268       0.10172 ±    0.01016       0.26971 ±    0.00457    11.818 ±  0.261 %    74.548 ±  0.482 %
+   9      22.7897 ±    0.7579       0.09567 ±    0.00959       0.26207 ±    0.00454    12.018 ±  0.261 %    75.627 ±  0.447 %
+  10      25.5304 ±    0.8149       0.09422 ±    0.00910       0.26722 ±    0.00423    11.875 ±  0.245 %    74.673 ±  0.430 %
+  11      22.6517 ±    0.6841       0.08713 ±    0.00868       0.26373 ±    0.00413    12.127 ±  0.239 %    75.473 ±  0.406 %
+  12      21.9205 ±    0.6291       0.09305 ±    0.00824       0.25784 ±    0.00386    12.107 ±  0.224 %    75.896 ±  0.386 %
+  13      21.8487 ±    0.6001       0.09711 ±    0.00794       0.26151 ±    0.00369    12.217 ±  0.212 %    75.389 ±  0.374 %
+  14      23.1626 ±    0.6156       0.09822 ±    0.00783       0.27225 ±    0.00376    12.233 ±  0.202 %    74.787 ±  0.363 %
+  15      22.3213 ±    0.5704       0.09799 ±    0.00755       0.26907 ±    0.00358    12.364 ±  0.194 %    74.852 ±  0.350 %
+  16      20.5861 ±    0.5066       0.09346 ±    0.00726       0.26292 ±    0.00350    12.466 ±  0.191 %    75.538 ±  0.336 %
+  17      20.4422 ±    0.4849       0.09092 ±    0.00705       0.26347 ±    0.00334    12.436 ±  0.182 %    75.418 ±  0.327 %
+  18      21.1229 ±    0.4864       0.09528 ±    0.00677       0.26083 ±    0.00318    12.345 ±  0.175 %    75.220 ±  0.318 %
+  19      21.4692 ±    0.4813       0.10350 ±    0.00664       0.26323 ±    0.00309    12.353 ±  0.169 %    75.073 ±  0.310 %
+  20      21.9715 ±    0.4797       0.10513 ±    0.00652       0.26749 ±    0.00300    12.335 ±  0.163 %    74.795 ±  0.304 %
+  21      22.3093 ±    0.4762       0.11283 ±    0.00637       0.26714 ±    0.00289    12.254 ±  0.158 %    74.831 ±  0.296 %
+  22      21.4290 ±    0.4442       0.11056 ±    0.00619       0.26275 ±    0.00288    12.236 ±  0.156 %    75.162 ±  0.288 %
+  23      20.2528 ±    0.4087       0.10778 ±    0.00599       0.25773 ±    0.00281    12.209 ±  0.153 %    75.520 ±  0.280 %
+  24      20.3877 ±    0.4037       0.10666 ±    0.00591       0.25995 ±    0.00278    12.223 ±  0.149 %    75.411 ±  0.275 %
+  25      19.6101 ±    0.3790       0.10507 ±    0.00572       0.25423 ±    0.00269    12.112 ±  0.145 %    75.844 ±  0.268 %
+  26      19.9641 ±    0.3780       0.10654 ±    0.00559       0.25261 ±    0.00260    12.072 ±  0.142 %    75.761 ±  0.263 %
+  27      18.9935 ±    0.3515       0.10465 ±    0.00548       0.25139 ±    0.00259    12.157 ±  0.140 %    76.025 ±  0.257 %
+  28      19.2026 ±    0.3484       0.10333 ±    0.00536       0.25101 ±    0.00253    12.132 ±  0.137 %    75.971 ±  0.252 %
+  29      19.5907 ±    0.3502       0.10597 ±    0.00529       0.25350 ±    0.00249    12.208 ±  0.135 %    75.798 ±  0.249 %
+  30      20.0606 ±    0.3525       0.10723 ±    0.00523       0.25618 ±    0.00246    12.194 ±  0.133 %    75.614 ±  0.245 %
+  31      19.8650 ±    0.3421       0.10679 ±    0.00514       0.25552 ±    0.00240    12.193 ±  0.130 %    75.600 ±  0.241 %
+  32      20.1615 ±    0.3426       0.13596 ±    0.00550       0.28938 ±    0.00299    13.417 ±  0.147 %    74.823 ±  0.240 %
+  33      20.4302 ±    0.3413       0.13522 ±    0.00541       0.28839 ±    0.00292    13.350 ±  0.144 %    74.819 ±  0.236 %
+  34      20.4814 ±    0.3362       0.13540 ±    0.00530       0.28669 ±    0.00284    13.315 ±  0.141 %    74.760 ±  0.233 %
+  35      20.7382 ±    0.3361       0.13478 ±    0.00524       0.28906 ��    0.00279    13.364 ±  0.138 %    74.571 ±  0.230 %
+  36      20.4010 ±    0.3249       0.09213 ±    0.00569       0.34290 ±    0.00388    14.213 ±  0.142 %    73.626 ±  0.230 %
+  37      20.6839 ±    0.3250       0.09383 ±    0.00557       0.33923 ±    0.00378    14.116 ±  0.140 %    73.626 ±  0.227 %
+  38      21.1550 ±    0.3287       0.09488 ±    0.00547       0.33794 ±    0.00370    14.049 ±  0.137 %    73.553 ±  0.224 %
+  39      21.4587 ±    0.3293       0.09731 ±    0.00537       0.33523 ±    0.00361    13.966 ±  0.135 %    73.587 ±  0.221 %
+  40      21.6332 ±    0.3279       0.09955 ±    0.00528       0.33395 ±    0.00353    13.916 ±  0.133 %    73.541 ±  0.218 %
+  41      21.9511 ±    0.3292       0.10213 ±    0.00521       0.33273 ±    0.00345    13.851 ±  0.131 %    73.421 ±  0.216 %
+  42      22.3018 ±    0.3313       0.10510 ±    0.00513       0.33257 ±    0.00338    13.804 ±  0.129 %    73.351 ±  0.213 %
+  43      22.6720 ±    0.3332       0.10673 ±    0.00507       0.33194 ±    0.00333    13.755 ±  0.127 %    73.307 ±  0.211 %
+  44      22.2382 ±    0.3220       0.10490 ±    0.00496       0.32583 ±    0.00326    13.630 ±  0.125 %    73.580 ±  0.208 %
+
+====== Perplexity statistics ======
+Mean PPL(Q)                   :  22.238185 ±   0.322012
+Mean PPL(base)                :  20.023532 ±   0.285215
+Cor(ln(PPL(Q)), ln(PPL(base))):  94.05%
+Mean ln(PPL(Q)/PPL(base))     :   0.104903 ±   0.004960
+Mean PPL(Q)/PPL(base)         :   1.110603 ±   0.005509
+Mean PPL(Q)-PPL(base)         :   2.214654 ±   0.110837
+
+====== KL divergence statistics ======
+Mean    KLD:   0.325828 ±   0.003258
+Maximum KLD:  19.564314
+99.9%   KLD:   8.606861
+99.0%   KLD:   3.195640
+95.0%   KLD:   1.148634
+90.0%   KLD:   0.695183
+Median  KLD:   0.149037
+10.0%   KLD:   0.009316
+ 5.0%   KLD:   0.001733
+ 1.0%   KLD:   0.000054
+ 0.1%   KLD:   0.000001
+Minimum KLD:  -0.000010
+
+====== Token probability statistics ======
+Mean    Δp: -1.203 ± 0.064 %
+Maximum Δp: 99.921%
+99.9%   Δp: 83.312%
+99.0%   Δp: 40.107%
+95.0%   Δp: 15.836%
+90.0%   Δp:  8.091%
+75.0%   Δp:  1.037%
+Median  Δp: -0.017%
+25.0%   Δp: -2.502%
+10.0%   Δp: -12.324%
+ 5.0%   Δp: -22.158%
+ 1.0%   Δp: -50.154%
+ 0.1%   Δp: -91.656%
+Minimum Δp: -99.845%
+RMS Δp    : 13.630 ± 0.125 %
+Same top p: 73.580 ± 0.208 %
+

reports/kld2048b-Q4_K_M.txt

@@ -0,0 +1,91 @@
+0.02.079.698 I 
+0.02.079.719 I system_info: n_threads = 12 (n_threads_batch = 12) / 16 | MTL : EMBED_LIBRARY = 1 | CPU : NEON = 1 | ARM_FMA = 1 | FP16_VA = 1 | MATMUL_INT8 = 1 | DOTPROD = 1 | SME = 1 | ACCELERATE = 1 | OPENMP = 1 | REPACK = 1 | 
+0.02.093.411 I kl_divergence: computing over 44 chunks, n_ctx=2048, batch_size=2048, n_seq=1
+0.06.162.087 I kl_divergence: 4.07 seconds per pass - ETA 2.98 minutes
+
+chunk             PPL               ln(PPL(Q)/PPL(base))          KL Divergence              Δp RMS            Same top p
+   1      41.2702 ±    4.1811       0.05150 ±    0.02115       0.14859 ±    0.00665     8.046 ±  0.420 %    78.592 ±  1.283 %
+   2      44.5736 ±    3.1803       0.02983 ±    0.01537       0.14332 ±    0.00463     8.115 ±  0.383 %    78.983 ±  0.901 %
+   3      31.3973 ±    1.8245       0.01533 ±    0.01133       0.11695 ±    0.00374     7.745 ±  0.367 %    82.470 ±  0.686 %
+   4      21.5957 ±    1.0734       0.01808 ±    0.00942       0.10860 ±    0.00433     7.784 ±  0.335 %    84.384 ±  0.568 %
+   5      16.9509 ±    0.7364       0.01266 ±    0.00797       0.09431 ±    0.00355     7.447 ±  0.290 %    86.276 ±  0.481 %
+   6      19.4643 ±    0.7848       0.02936 ±    0.00782       0.10815 ±    0.00334     7.874 ±  0.266 %    84.848 ±  0.458 %
+   7      22.6163 ±    0.8587       0.05597 ±    0.00756       0.11632 ±    0.00305     7.995 ±  0.240 %    83.689 ±  0.437 %
+   8      20.5296 ±    0.7191      -0.13907 ±    0.01298       0.46931 ±    0.01663    15.957 ±  0.400 %    78.372 ±  0.455 %
+   9      18.3012 ±    0.5997      -0.12367 ±    0.01168       0.42440 ±    0.01486    15.241 ±  0.375 %    79.668 ±  0.419 %
+  10      20.9340 ±    0.6618      -0.10428 ±    0.01075       0.39735 ±    0.01342    14.637 ±  0.354 %    79.492 ±  0.399 %
+  11      18.9142 ±    0.5668      -0.09319 ±    0.00991       0.36823 ±    0.01224    14.119 ±  0.334 %    80.494 ±  0.374 %
+  12      19.0460 ±    0.5453      -0.04751 ±    0.00953       0.36436 ±    0.01130    14.121 ±  0.314 %    80.173 ±  0.360 %
+  13      18.9689 ±    0.5190      -0.04423 ±    0.00891       0.34501 ±    0.01046    13.764 ±  0.298 %    80.232 ±  0.345 %
+  14      20.2610 ±    0.5377      -0.03562 ±    0.00840       0.33139 ±    0.00973    13.449 ±  0.284 %    80.052 ±  0.334 %
+  15      19.6396 ±    0.5014      -0.03001 ±    0.00796       0.31598 ±    0.00910    13.229 ±  0.271 %    80.391 ±  0.321 %
+  16      18.2437 ±    0.4482      -0.02734 ±    0.00755       0.30048 ±    0.00857    12.932 ±  0.261 %    81.103 ±  0.306 %
+  17      18.1996 ±    0.4304      -0.02528 ±    0.00719       0.28990 ±    0.00808    12.683 ±  0.251 %    81.111 ±  0.297 %
+  18      18.7860 ±    0.4310      -0.02197 ±    0.00685       0.27922 ±    0.00764    12.440 ±  0.242 %    81.139 ±  0.288 %
+  19      19.0784 ±    0.4252      -0.01457 ±    0.00659       0.27199 ±    0.00725    12.275 ±  0.233 %    81.062 ±  0.281 %
+  20      19.6134 ±    0.4257      -0.00840 ±    0.00634       0.26519 ±    0.00689    12.079 ±  0.226 %    81.061 ±  0.274 %
+  21      19.8939 ±    0.4212      -0.00176 ±    0.00610       0.25827 ±    0.00657    11.900 ±  0.219 %    81.120 ±  0.267 %
+  22      19.1779 ±    0.3941      -0.00042 ±    0.00585       0.24875 ±    0.00628    11.704 ±  0.213 %    81.565 ±  0.258 %
+  23      18.1599 ±    0.3630      -0.00130 ±    0.00562       0.23988 ±    0.00602    11.510 ±  0.207 %    81.992 ±  0.251 %
+  24      18.3523 ±    0.3604       0.00148 ±    0.00548       0.23611 ±    0.00578    11.406 ±  0.201 %    81.965 ±  0.245 %
+  25      17.6893 ±    0.3389       0.00199 ±    0.00528       0.22815 ±    0.00556    11.223 ±  0.196 %    82.350 ±  0.238 %
+  26      18.0398 ±    0.3388       0.00519 ±    0.00512       0.22323 ±    0.00535    11.094 ±  0.191 %    82.326 ±  0.234 %
+  27      17.2077 ±    0.3162       0.00591 ±    0.00498       0.21861 ±    0.00516    11.067 ±  0.186 %    82.604 ±  0.228 %
+  28      17.4397 ±    0.3143       0.00703 ±    0.00484       0.21471 ±    0.00498    10.962 ±  0.181 %    82.625 ±  0.224 %
+  29      17.8198 ±    0.3168       0.01122 ±    0.00474       0.21251 ±    0.00481    10.920 ±  0.177 %    82.502 ±  0.221 %
+  30      18.2178 ±    0.3180       0.01087 ±    0.00464       0.21008 ±    0.00466    10.842 ±  0.173 %    82.398 ±  0.217 %
+  31      18.0640 ±    0.3089       0.01176 ±    0.00452       0.20629 ±    0.00451    10.772 ±  0.169 %    82.461 ±  0.214 %
+  32      17.6394 ±    0.2959       0.00232 ±    0.00456       0.21097 ±    0.00444    11.023 ±  0.166 %    82.310 ±  0.211 %
+  33      17.9033 ±    0.2954       0.00318 ±    0.00446       0.20772 ±    0.00431    10.918 ±  0.163 %    82.254 ±  0.208 %
+  34      17.9506 ±    0.2909       0.00351 ±    0.00436       0.20459 ±    0.00418    10.853 ±  0.159 %    82.238 ±  0.205 %
+  35      18.2135 ±    0.2917       0.00497 ±    0.00428       0.20221 ��    0.00407    10.793 ±  0.156 %    82.265 ±  0.202 %
+  36      18.6417 ±    0.2959       0.00195 ±    0.00464       0.24893 ±    0.00457    11.471 ±  0.157 %    81.194 ±  0.204 %
+  37      18.8949 ±    0.2959       0.00337 ±    0.00454       0.24459 ±    0.00445    11.358 ±  0.154 %    81.319 ±  0.200 %
+  38      19.3157 ±    0.2992       0.00392 ±    0.00444       0.24126 ±    0.00433    11.275 ±  0.152 %    81.275 ±  0.198 %
+  39      19.5740 ±    0.2992       0.00539 ±    0.00435       0.23784 ±    0.00422    11.198 ±  0.149 %    81.257 ±  0.195 %
+  40      19.7017 ±    0.2971       0.00603 ±    0.00426       0.23443 ±    0.00412    11.108 ±  0.147 %    81.261 ±  0.193 %
+  41      19.9681 ±    0.2978       0.00745 ±    0.00418       0.23180 ±    0.00402    11.039 ±  0.144 %    81.189 ±  0.191 %
+  42      20.2624 ±    0.2992       0.00920 ±    0.00411       0.22991 ±    0.00393    10.986 ±  0.142 %    81.141 ±  0.189 %
+  43      20.5921 ±    0.3007       0.01051 ±    0.00404       0.22778 ±    0.00384    10.918 ±  0.139 %    81.125 ±  0.187 %
+  44      20.2269 ±    0.2910       0.01010 ±    0.00396       0.22299 ±    0.00376    10.805 ±  0.138 %    81.387 ±  0.183 %
+
+====== Perplexity statistics ======
+Mean PPL(Q)                   :  20.226883 ±   0.291005
+Mean PPL(base)                :  20.023532 ±   0.285215
+Cor(ln(PPL(Q)), ln(PPL(base))):  96.19%
+Mean ln(PPL(Q)/PPL(base))     :   0.010104 ±   0.003955
+Mean PPL(Q)/PPL(base)         :   1.010156 ±   0.003995
+Mean PPL(Q)-PPL(base)         :   0.203352 ±   0.079759
+
+====== KL divergence statistics ======
+Mean    KLD:   0.222994 ±   0.003760
+Maximum KLD:  21.454760
+99.9%   KLD:  11.217873
+99.0%   KLD:   3.457192
+95.0%   KLD:   0.742914
+90.0%   KLD:   0.354640
+Median  KLD:   0.061749
+10.0%   KLD:   0.003209
+ 5.0%   KLD:   0.000565
+ 1.0%   KLD:   0.000016
+ 0.1%   KLD:   0.000000
+Minimum KLD:  -0.000076
+
+====== Token probability statistics ======
+Mean    Δp:  0.222 ± 0.051 %
+Maximum Δp: 99.968%
+99.9%   Δp: 97.018%
+99.0%   Δp: 39.167%
+95.0%   Δp: 12.490%
+90.0%   Δp:  6.425%
+75.0%   Δp:  0.990%
+Median  Δp: -0.002%
+25.0%   Δp: -1.192%
+10.0%   Δp: -6.597%
+ 5.0%   Δp: -12.478%
+ 1.0%   Δp: -31.393%
+ 0.1%   Δp: -65.134%
+Minimum Δp: -91.785%
+RMS Δp    : 10.805 ± 0.138 %
+Same top p: 81.387 ± 0.183 %
+

reports/kld2048b-Q5_K_M.txt

@@ -0,0 +1,91 @@
+0.00.692.086 I 
+0.00.692.109 I system_info: n_threads = 12 (n_threads_batch = 12) / 16 | MTL : EMBED_LIBRARY = 1 | CPU : NEON = 1 | ARM_FMA = 1 | FP16_VA = 1 | MATMUL_INT8 = 1 | DOTPROD = 1 | SME = 1 | ACCELERATE = 1 | OPENMP = 1 | REPACK = 1 | 
+0.00.707.153 I kl_divergence: computing over 44 chunks, n_ctx=2048, batch_size=2048, n_seq=1
+0.05.303.346 I kl_divergence: 4.60 seconds per pass - ETA 3.37 minutes
+
+chunk             PPL               ln(PPL(Q)/PPL(base))          KL Divergence              Δp RMS            Same top p
+   1      39.9630 ±    4.0696       0.01932 ±    0.01341       0.05495 ±    0.00231     5.203 ±  0.376 %    86.217 ±  1.078 %
+   2      43.5242 ±    3.1197       0.00601 ±    0.01041       0.06186 ±    0.00216     5.467 ±  0.266 %    85.386 ±  0.781 %
+   3      30.9644 ±    1.8103       0.00145 ±    0.00731       0.04755 ±    0.00155     4.832 ±  0.211 %    88.302 ±  0.580 %
+   4      21.1607 ±    1.0531      -0.00227 ±    0.00611       0.04406 ±    0.00272     4.902 ±  0.299 %    89.370 ±  0.482 %
+   5      16.6963 ±    0.7261      -0.00247 ±    0.00511       0.03802 ±    0.00223     4.682 ±  0.268 %    90.694 ±  0.406 %
+   6      18.8603 ±    0.7547      -0.00216 ±    0.00504       0.04566 ±    0.00203     5.261 ±  0.258 %    89.541 ±  0.391 %
+   7      21.5013 ±    0.8064       0.00541 ±    0.00476       0.04762 ±    0.00179     5.240 ±  0.228 %    89.317 ±  0.365 %
+   8      23.7471 ±    0.8336       0.00652 ±    0.00456       0.05059 ±    0.00162     5.252 ±  0.205 %    88.685 ±  0.350 %
+   9      20.8119 ±    0.6849       0.00489 ±    0.00421       0.04720 ±    0.00147     5.094 ±  0.190 %    89.399 ±  0.321 %
+  10      23.2395 ±    0.7341       0.00020 ±    0.00401       0.04927 ±    0.00136     5.073 ±  0.176 %    88.915 ±  0.310 %
+  11      20.7576 ±    0.6221      -0.00019 ±    0.00379       0.04718 ±    0.00126     5.073 ±  0.166 %    89.514 ±  0.289 %
+  12      19.8390 ±    0.5638      -0.00673 ±    0.00362       0.04730 ±    0.00118     5.261 ±  0.159 %    89.589 ±  0.276 %
+  13      19.7641 ±    0.5371      -0.00316 ±    0.00347       0.04832 ±    0.00112     5.362 ±  0.152 %    89.338 ±  0.268 %
+  14      20.8728 ±    0.5490      -0.00587 ±    0.00341       0.05021 ±    0.00107     5.379 ±  0.142 %    88.807 ±  0.263 %
+  15      20.1173 ±    0.5087      -0.00597 ±    0.00331       0.05014 ±    0.00102     5.454 ±  0.134 %    88.895 ±  0.254 %
+  16      18.6445 ±    0.4542      -0.00560 ±    0.00314       0.04803 ±    0.00097     5.374 ±  0.129 %    89.266 ±  0.242 %
+  17      18.5453 ±    0.4345      -0.00646 ±    0.00302       0.04736 ±    0.00091     5.327 ±  0.123 %    89.265 ±  0.235 %
+  18      19.0987 ±    0.4338      -0.00546 ±    0.00289       0.04654 ±    0.00087     5.293 ±  0.118 %    89.302 ±  0.228 %
+  19      19.3034 ±    0.4256      -0.00284 ±    0.00282       0.04721 ±    0.00083     5.325 ±  0.113 %    89.165 ±  0.223 %
+  20      19.7218 ±    0.4225      -0.00289 ±    0.00276       0.04737 ±    0.00080     5.303 ±  0.109 %    89.115 ±  0.218 %
+  21      19.8754 ±    0.4148      -0.00269 ±    0.00267       0.04721 ±    0.00077     5.272 ±  0.105 %    89.168 ±  0.212 %
+  22      19.1387 ±    0.3879      -0.00247 ±    0.00257       0.04578 ±    0.00074     5.198 ±  0.102 %    89.429 ±  0.205 %
+  23      18.1283 ±    0.3576      -0.00304 ±    0.00248       0.04464 ±    0.00074     5.156 ±  0.101 %    89.715 ±  0.198 %
+  24      18.2849 ±    0.3541      -0.00220 ±    0.00245       0.04493 ±    0.00073     5.192 ±  0.098 %    89.622 ±  0.195 %
+  25      17.6103 ±    0.3329      -0.00249 ±    0.00236       0.04363 ±    0.00070     5.118 ±  0.095 %    89.822 ±  0.189 %
+  26      17.9036 ±    0.3319      -0.00239 ±    0.00232       0.04353 ±    0.00068     5.085 ±  0.093 %    89.785 ±  0.186 %
+  27      17.0832 ±    0.3098      -0.00135 ±    0.00229       0.04387 ±    0.00069     5.237 ±  0.096 %    89.870 ±  0.182 %
+  28      17.2934 ±    0.3076      -0.00139 ±    0.00224       0.04378 ±    0.00067     5.228 ±  0.094 %    89.862 ±  0.178 %
+  29      17.5885 ±    0.3082      -0.00184 ±    0.00222       0.04429 ±    0.00065     5.250 ±  0.091 %    89.770 ±  0.176 %
+  30      17.9869 ±    0.3097      -0.00188 ±    0.00219       0.04490 ±    0.00064     5.247 ±  0.089 %    89.642 ±  0.174 %
+  31      17.8276 ±    0.3007      -0.00142 ±    0.00215       0.04470 ±    0.00062     5.253 ±  0.088 %    89.686 ±  0.171 %
+  32      17.4479 ±    0.2893      -0.00859 ±    0.00233       0.05061 ±    0.00086     5.790 ±  0.106 %    89.421 ±  0.170 %
+  33      17.6847 ±    0.2885      -0.00910 ±    0.00229       0.05052 ±    0.00084     5.774 ±  0.103 %    89.369 ±  0.168 %
+  34      17.7162 ±    0.2839      -0.00964 ±    0.00225       0.05027 ±    0.00082     5.754 ±  0.101 %    89.293 ±  0.166 %
+  35      17.9647 ±    0.2843      -0.00879 ±    0.00222       0.05104 ��    0.00081     5.788 ±  0.099 %    89.180 ±  0.164 %
+  36      18.4521 ±    0.2895      -0.00827 ±    0.00224       0.05340 ±    0.00082     5.846 ±  0.096 %    88.938 ±  0.163 %
+  37      18.6706 ±    0.2888      -0.00858 ±    0.00220       0.05283 ±    0.00080     5.802 ±  0.094 %    88.999 ±  0.161 %
+  38      19.0670 ±    0.2914      -0.00904 ±    0.00216       0.05264 ±    0.00078     5.783 ±  0.092 %    88.946 ±  0.159 %
+  39      19.2958 ±    0.2909      -0.00893 ±    0.00212       0.05232 ±    0.00076     5.754 ±  0.091 %    88.937 ±  0.157 %
+  40      19.3982 ±    0.2885      -0.00950 ±    0.00208       0.05215 ±    0.00074     5.724 ±  0.089 %    88.917 ±  0.155 %
+  41      19.6150 ±    0.2883      -0.01039 ±    0.00205       0.05202 ±    0.00073     5.699 ±  0.087 %    88.813 ±  0.154 %
+  42      19.8756 ±    0.2890      -0.01008 ±    0.00203       0.05223 ±    0.00071     5.695 ±  0.086 %    88.773 ±  0.152 %
+  43      20.1426 ±    0.2895      -0.01156 ±    0.00200       0.05217 ±    0.00070     5.676 ±  0.084 %    88.743 ±  0.151 %
+  44      19.7970 ±    0.2804      -0.01138 ±    0.00196       0.05113 ±    0.00068     5.620 ±  0.083 %    88.887 ±  0.148 %
+
+====== Perplexity statistics ======
+Mean PPL(Q)                   :  19.797018 ±   0.280375
+Mean PPL(base)                :  20.023532 ±   0.285215
+Cor(ln(PPL(Q)), ln(PPL(base))):  99.05%
+Mean ln(PPL(Q)/PPL(base))     :  -0.011377 ±   0.001959
+Mean PPL(Q)/PPL(base)         :   0.988688 ±   0.001937
+Mean PPL(Q)-PPL(base)         :  -0.226513 ±   0.039268
+
+====== KL divergence statistics ======
+Mean    KLD:   0.051125 ±   0.000685
+Maximum KLD:   9.869919
+99.9%   KLD:   1.489997
+99.0%   KLD:   0.466226
+95.0%   KLD:   0.174651
+90.0%   KLD:   0.109054
+Median  KLD:   0.022373
+10.0%   KLD:   0.001169
+ 5.0%   KLD:   0.000194
+ 1.0%   KLD:   0.000005
+ 0.1%   KLD:  -0.000001
+Minimum KLD:  -0.000079
+
+====== Token probability statistics ======
+Mean    Δp: -0.136 ± 0.026 %
+Maximum Δp: 99.862%
+99.9%   Δp: 44.541%
+99.0%   Δp: 17.103%
+95.0%   Δp:  6.634%
+90.0%   Δp:  3.349%
+75.0%   Δp:  0.531%
+Median  Δp: -0.000%
+25.0%   Δp: -0.740%
+10.0%   Δp: -4.072%
+ 5.0%   Δp: -7.444%
+ 1.0%   Δp: -18.135%
+ 0.1%   Δp: -39.395%
+Minimum Δp: -82.381%
+RMS Δp    :  5.620 ± 0.083 %
+Same top p: 88.887 ± 0.148 %
+

reports/kld2048b-Q6_K.txt

@@ -0,0 +1,91 @@
+0.00.791.202 I 
+0.00.791.226 I system_info: n_threads = 12 (n_threads_batch = 12) / 16 | MTL : EMBED_LIBRARY = 1 | CPU : NEON = 1 | ARM_FMA = 1 | FP16_VA = 1 | MATMUL_INT8 = 1 | DOTPROD = 1 | SME = 1 | ACCELERATE = 1 | OPENMP = 1 | REPACK = 1 | 
+0.00.806.466 I kl_divergence: computing over 44 chunks, n_ctx=2048, batch_size=2048, n_seq=1
+0.06.581.283 I kl_divergence: 5.77 seconds per pass - ETA 4.23 minutes
+
+chunk             PPL               ln(PPL(Q)/PPL(base))          KL Divergence              Δp RMS            Same top p
+   1      39.7693 ±    4.0512       0.01446 ±    0.01296       0.04919 ±    0.00219     4.913 ±  0.344 %    85.826 ±  1.091 %
+   2      42.9960 ±    3.0837      -0.00621 ±    0.00972       0.05312 ±    0.00169     4.983 ±  0.248 %    86.315 ±  0.760 %
+   3      30.7245 ±    1.7950      -0.00633 ±    0.00679       0.03963 ±    0.00120     4.331 ±  0.194 %    89.084 ±  0.563 %
+   4      21.0200 ±    1.0464      -0.00894 ±    0.00557       0.03574 ±    0.00228     4.472 ±  0.310 %    90.591 ±  0.456 %
+   5      16.6222 ±    0.7230      -0.00692 ±    0.00456       0.02987 ±    0.00184     4.137 ±  0.270 %    91.945 ±  0.381 %
+   6      18.8735 ±    0.7573      -0.00146 ±    0.00443       0.03418 ±    0.00161     4.531 ±  0.235 %    91.088 ±  0.364 %
+   7      21.3666 ±    0.8029      -0.00087 ±    0.00432       0.03786 ±    0.00147     4.667 ±  0.211 %    90.476 ±  0.347 %
+   8      23.5987 ±    0.8302       0.00025 ±    0.00420       0.04052 ±    0.00133     4.676 ±  0.187 %    89.907 ±  0.333 %
+   9      20.6999 ±    0.6823      -0.00051 ±    0.00384       0.03759 ±    0.00119     4.502 ±  0.173 %    90.453 ±  0.306 %
+  10      23.0941 ±    0.7301      -0.00607 ±    0.00366       0.03977 ±    0.00111     4.498 ±  0.160 %    89.814 ±  0.299 %
+  11      20.6333 ±    0.6188      -0.00620 ±    0.00341       0.03763 ±    0.00102     4.404 ±  0.149 %    90.411 ±  0.278 %
+  12      19.9340 ±    0.5679      -0.00194 ±    0.00335       0.04076 ±    0.00099     4.744 ±  0.143 %    90.176 ±  0.269 %
+  13      19.8184 ±    0.5400      -0.00042 ±    0.00320       0.04112 ±    0.00093     4.836 ±  0.136 %    90.105 ±  0.259 %
+  14      20.8748 ±    0.5505      -0.00578 ±    0.00316       0.04311 ±    0.00090     4.876 ±  0.128 %    89.652 ±  0.255 %
+  15      20.1217 ±    0.5101      -0.00575 ±    0.00307       0.04303 ±    0.00085     4.975 ±  0.121 %    89.697 ±  0.245 %
+  16      18.6525 ±    0.4555      -0.00517 ±    0.00290       0.04100 ±    0.00081     4.884 ±  0.116 %    90.090 ±  0.234 %
+  17      18.5724 ±    0.4363      -0.00500 ±    0.00279       0.04037 ±    0.00077     4.825 ±  0.111 %    90.087 ±  0.227 %
+  18      19.0767 ±    0.4345      -0.00661 ±    0.00268       0.03954 ±    0.00073     4.780 ±  0.107 %    90.187 ±  0.219 %
+  19      19.2489 ±    0.4255      -0.00567 ±    0.00261       0.03969 ±    0.00070     4.786 ±  0.102 %    90.096 ±  0.214 %
+  20      19.6439 ±    0.4222      -0.00685 ±    0.00254       0.03964 ±    0.00067     4.754 ±  0.098 %    90.049 ±  0.209 %
+  21      19.7655 ±    0.4138      -0.00824 ±    0.00246       0.03914 ±    0.00064     4.718 ±  0.095 %    90.113 ±  0.204 %
+  22      19.0431 ±    0.3872      -0.00748 ±    0.00236       0.03785 ±    0.00061     4.647 ±  0.092 %    90.354 ±  0.197 %
+  23      18.0370 ±    0.3568      -0.00809 ±    0.00227       0.03679 ±    0.00060     4.609 ±  0.092 %    90.565 ±  0.191 %
+  24      18.1912 ±    0.3532      -0.00733 ±    0.00225       0.03715 ±    0.00059     4.656 ±  0.090 %    90.441 ±  0.188 %
+  25      17.5234 ±    0.3320      -0.00744 ±    0.00217       0.03602 ±    0.00056     4.592 ±  0.088 %    90.663 ±  0.182 %
+  26      17.7978 ±    0.3305      -0.00831 ±    0.00212       0.03589 ±    0.00054     4.584 ±  0.086 %    90.627 ±  0.179 %
+  27      16.9643 ±    0.3080      -0.00833 ±    0.00209       0.03595 ±    0.00055     4.681 ±  0.088 %    90.743 ±  0.174 %
+  28      17.1752 ±    0.3058      -0.00825 ±    0.00204       0.03580 ±    0.00053     4.683 ±  0.085 %    90.738 ±  0.171 %
+  29      17.4899 ±    0.3070      -0.00746 ±    0.00202       0.03632 ±    0.00052     4.704 ±  0.083 %    90.633 ±  0.169 %
+  30      17.8846 ±    0.3083      -0.00759 ±    0.00200       0.03698 ±    0.00051     4.713 ±  0.080 %    90.505 ±  0.167 %
+  31      17.7369 ±    0.2995      -0.00652 ±    0.00197       0.03677 ±    0.00050     4.729 ±  0.080 %    90.540 ±  0.164 %
+  32      17.5366 ±    0.2916      -0.00352 ±    0.00196       0.03685 ±    0.00049     4.777 ±  0.079 %    90.558 ±  0.162 %
+  33      17.7734 ±    0.2908      -0.00410 ±    0.00193       0.03695 ±    0.00048     4.767 ±  0.077 %    90.471 ±  0.160 %
+  34      17.8213 ±    0.2865      -0.00372 ±    0.00189       0.03687 ±    0.00047     4.763 ±  0.076 %    90.363 ±  0.158 %
+  35      18.0669 ±    0.2869      -0.00312 ±    0.00187       0.03753 ��    0.00048     4.800 ±  0.075 %    90.292 ±  0.156 %
+  36      18.5795 ±    0.2929      -0.00139 ±    0.00278       0.08702 ±    0.00233     6.398 ±  0.131 %    89.011 ±  0.163 %
+  37      18.8027 ±    0.2923      -0.00152 ±    0.00271       0.08538 ±    0.00227     6.337 ±  0.129 %    89.118 ±  0.160 %
+  38      19.1927 ±    0.2947      -0.00247 ±    0.00265       0.08427 ±    0.00221     6.297 ±  0.127 %    89.036 ±  0.158 %
+  39      19.4273 ±    0.2944      -0.00214 ±    0.00260       0.08312 ±    0.00216     6.252 ±  0.124 %    88.999 ±  0.157 %
+  40      19.5252 ±    0.2918      -0.00297 ±    0.00255       0.08206 ±    0.00210     6.210 ±  0.122 %    88.969 ±  0.155 %
+  41      19.7581 ±    0.2919      -0.00312 ±    0.00250       0.08099 ±    0.00205     6.170 ±  0.120 %    88.964 ±  0.153 %
+  42      20.0080 ±    0.2924      -0.00344 ±    0.00246       0.08034 ±    0.00200     6.150 ±  0.118 %    88.905 ±  0.152 %
+  43      20.2833 ±    0.2930      -0.00460 ±    0.00242       0.07954 ±    0.00196     6.114 ±  0.116 %    88.895 ±  0.150 %
+  44      19.9327 ±    0.2838      -0.00455 ±    0.00236       0.07786 ±    0.00192     6.050 ±  0.114 %    89.056 ±  0.147 %
+
+====== Perplexity statistics ======
+Mean PPL(Q)                   :  19.932671 ±   0.283759
+Mean PPL(base)                :  20.023532 ±   0.285215
+Cor(ln(PPL(Q)), ln(PPL(base))):  98.62%
+Mean ln(PPL(Q)/PPL(base))     :  -0.004548 ±   0.002363
+Mean PPL(Q)/PPL(base)         :   0.995462 ±   0.002352
+Mean PPL(Q)-PPL(base)         :  -0.090860 ±   0.047230
+
+====== KL divergence statistics ======
+Mean    KLD:   0.077855 ±   0.001915
+Maximum KLD:  16.764179
+99.9%   KLD:   6.203948
+99.0%   KLD:   1.520848
+95.0%   KLD:   0.174014
+90.0%   KLD:   0.098078
+Median  KLD:   0.018081
+10.0%   KLD:   0.000850
+ 5.0%   KLD:   0.000131
+ 1.0%   KLD:   0.000003
+ 0.1%   KLD:  -0.000001
+Minimum KLD:  -0.000081
+
+====== Token probability statistics ======
+Mean    Δp:  0.020 ± 0.029 %
+Maximum Δp: 99.854%
+99.9%   Δp: 62.315%
+99.0%   Δp: 16.727%
+95.0%   Δp:  6.435%
+90.0%   Δp:  3.405%
+75.0%   Δp:  0.568%
+Median  Δp:  0.000%
+25.0%   Δp: -0.562%
+10.0%   Δp: -3.505%
+ 5.0%   Δp: -6.514%
+ 1.0%   Δp: -16.873%
+ 0.1%   Δp: -49.872%
+Minimum Δp: -97.627%
+RMS Δp    :  6.050 ± 0.114 %
+Same top p: 89.056 ± 0.147 %
+

reports/kld2048b-Q8_0.txt

@@ -0,0 +1,91 @@
+0.03.902.859 I 
+0.03.902.882 I system_info: n_threads = 12 (n_threads_batch = 12) / 16 | MTL : EMBED_LIBRARY = 1 | CPU : NEON = 1 | ARM_FMA = 1 | FP16_VA = 1 | MATMUL_INT8 = 1 | DOTPROD = 1 | SME = 1 | ACCELERATE = 1 | OPENMP = 1 | REPACK = 1 | 
+0.03.917.638 I kl_divergence: computing over 44 chunks, n_ctx=2048, batch_size=2048, n_seq=1
+0.06.676.459 I kl_divergence: 2.76 seconds per pass - ETA 2.02 minutes
+
+chunk             PPL               ln(PPL(Q)/PPL(base))          KL Divergence              Δp RMS            Same top p
+   1      39.3179 ±    3.9687       0.00304 ±    0.00447       0.00512 ±    0.00020     1.719 ±  0.116 %    95.992 ±  0.614 %
+   2      43.7425 ±    3.1358       0.01101 ±    0.00482       0.01071 ±    0.00056     2.402 ±  0.135 %    94.379 ±  0.509 %
+   3      31.1566 ±    1.8236       0.00764 ±    0.00333       0.00775 ±    0.00040     2.043 ±  0.108 %    95.797 ±  0.362 %
+   4      21.3343 ±    1.0652       0.00590 ±    0.00256       0.00627 ±    0.00030     1.885 ±  0.090 %    96.359 ±  0.293 %
+   5      16.8234 ±    0.7337       0.00512 ±    0.00210       0.00519 ±    0.00025     1.726 ±  0.079 %    96.950 ±  0.240 %
+   6      19.0176 ±    0.7650       0.00615 ±    0.00192       0.00556 ±    0.00022     1.787 ±  0.071 %    96.693 ±  0.228 %
+   7      21.5324 ±    0.8110       0.00686 ±    0.00177       0.00567 ±    0.00019     1.783 ±  0.064 %    96.411 ±  0.220 %
+   8      23.7777 ±    0.8379       0.00781 ±    0.00163       0.00568 ±    0.00017     1.738 ±  0.058 %    96.261 ±  0.210 %
+   9      20.8430 ±    0.6883       0.00638 ±    0.00148       0.00523 ±    0.00015     1.671 ±  0.054 %    96.470 ±  0.192 %
+  10      23.4044 ±    0.7421       0.00727 ±    0.00140       0.00541 ±    0.00016     1.658 ±  0.050 %    96.325 ±  0.186 %
+  11      20.9095 ±    0.6296       0.00710 ±    0.00132       0.00510 ±    0.00015     1.624 ±  0.047 %    96.561 ±  0.172 %
+  12      20.1019 ±    0.5738       0.00644 ±    0.00124       0.00504 ±    0.00014     1.643 ±  0.045 %    96.562 ±  0.164 %
+  13      19.9460 ±    0.5443       0.00600 ±    0.00118       0.00507 ±    0.00013     1.675 ±  0.042 %    96.534 ±  0.159 %
+  14      21.1268 ±    0.5585       0.00622 ±    0.00116       0.00532 ±    0.00013     1.688 ±  0.040 %    96.327 ±  0.157 %
+  15      20.3644 ±    0.5177       0.00623 ±    0.00111       0.00519 ±    0.00012     1.697 ±  0.038 %    96.357 ±  0.151 %
+  16      18.8566 ±    0.4618       0.00571 ±    0.00105       0.00494 ±    0.00011     1.667 ±  0.037 %    96.518 ±  0.143 %
+  17      18.7770 ±    0.4422       0.00596 ±    0.00101       0.00486 ±    0.00011     1.653 ±  0.035 %    96.527 ±  0.139 %
+  18      19.3249 ±    0.4413       0.00632 ±    0.00096       0.00475 ±    0.00010     1.634 ±  0.034 %    96.573 ±  0.134 %
+  19      19.4971 ±    0.4323       0.00714 ±    0.00093       0.00471 ±    0.00010     1.629 ±  0.032 %    96.584 ±  0.130 %
+  20      19.9216 ±    0.4295       0.00719 ±    0.00090       0.00470 ±    0.00009     1.623 ±  0.031 %    96.608 ±  0.127 %
+  21      20.0633 ±    0.4215       0.00672 ±    0.00087       0.00462 ±    0.00009     1.614 ±  0.031 %    96.639 ±  0.123 %
+  22      19.3105 ±    0.3940       0.00646 ±    0.00084       0.00448 ±    0.00008     1.589 ±  0.030 %    96.716 ±  0.119 %
+  23      18.2955 ±    0.3633       0.00614 ±    0.00080       0.00433 ±    0.00008     1.565 ±  0.029 %    96.791 ±  0.115 %
+  24      18.4489 ±    0.3596       0.00673 ±    0.00081       0.00434 ±    0.00008     1.577 ±  0.028 %    96.790 ±  0.112 %
+  25      17.7665 ±    0.3379       0.00634 ±    0.00078       0.00421 ±    0.00007     1.556 ±  0.028 %    96.876 ±  0.109 %
+  26      18.0634 ±    0.3369       0.00650 ±    0.00076       0.00423 ±    0.00007     1.570 ±  0.027 %    96.857 ±  0.107 %
+  27      17.2203 ±    0.3140       0.00664 ±    0.00076       0.00431 ±    0.00008     1.617 ±  0.031 %    96.854 ±  0.105 %
+  28      17.4329 ±    0.3118       0.00664 ±    0.00074       0.00427 ±    0.00007     1.609 ±  0.030 %    96.837 ±  0.103 %
+  29      17.7293 ±    0.3125       0.00613 ±    0.00074       0.00435 ±    0.00007     1.622 ±  0.029 %    96.794 ±  0.102 %
+  30      18.1300 ±    0.3139       0.00604 ±    0.00073       0.00440 ±    0.00007     1.617 ±  0.028 %    96.774 ±  0.101 %
+  31      17.9560 ±    0.3043       0.00576 ±    0.00071       0.00436 ±    0.00007     1.617 ±  0.028 %    96.806 ±  0.099 %
+  32      17.7479 ±    0.2960       0.00845 ±    0.00077       0.00503 ±    0.00009     1.865 ±  0.047 %    96.722 ±  0.098 %
+  33      18.0002 ±    0.2953       0.00858 ±    0.00075       0.00500 ±    0.00009     1.857 ±  0.046 %    96.673 ±  0.098 %
+  34      18.0432 ±    0.2910       0.00865 ±    0.00074       0.00496 ±    0.00008     1.848 ±  0.045 %    96.651 ±  0.096 %
+  35      18.2803 ±    0.2911       0.00863 ±    0.00073       0.00500 ��    0.00008     1.848 ±  0.044 %    96.612 ±  0.096 %
+  36      18.7684 ±    0.2961       0.00872 ±    0.00079       0.00611 ±    0.00011     1.952 ±  0.044 %    96.416 ±  0.097 %
+  37      18.9962 ±    0.2956       0.00872 ±    0.00078       0.00603 ±    0.00011     1.934 ±  0.043 %    96.418 ±  0.096 %
+  38      19.4098 ±    0.2985       0.00878 ±    0.00076       0.00599 ±    0.00011     1.922 ±  0.042 %    96.393 ±  0.095 %
+  39      19.6427 ±    0.2980       0.00889 ±    0.00075       0.00593 ±    0.00011     1.911 ±  0.042 %    96.393 ±  0.093 %
+  40      19.7654 ±    0.2958       0.00925 ±    0.00073       0.00590 ±    0.00010     1.904 ±  0.041 %    96.378 ±  0.092 %
+  41      20.0024 ±    0.2960       0.00917 ±    0.00072       0.00587 ±    0.00010     1.894 ±  0.040 %    96.366 ±  0.091 %
+  42      20.2606 ±    0.2967       0.00911 ±    0.00071       0.00585 ±    0.00010     1.887 ±  0.039 %    96.348 ±  0.090 %
+  43      20.5612 ±    0.2977       0.00900 ±    0.00070       0.00583 ±    0.00010     1.879 ±  0.038 %    96.329 ±  0.090 %
+  44      20.2006 ±    0.2882       0.00880 ±    0.00069       0.00571 ±    0.00009     1.860 ±  0.038 %    96.381 ±  0.088 %
+
+====== Perplexity statistics ======
+Mean PPL(Q)                   :  20.200587 ±   0.288209
+Mean PPL(base)                :  20.023532 ±   0.285215
+Cor(ln(PPL(Q)), ln(PPL(base))):  99.88%
+Mean ln(PPL(Q)/PPL(base))     :   0.008803 ±   0.000686
+Mean PPL(Q)/PPL(base)         :   1.008842 ±   0.000692
+Mean PPL(Q)-PPL(base)         :   0.177055 ±   0.014106
+
+====== KL divergence statistics ======
+Mean    KLD:   0.005708 ±   0.000095
+Maximum KLD:   1.475488
+99.9%   KLD:   0.233686
+99.0%   KLD:   0.060422
+95.0%   KLD:   0.019779
+90.0%   KLD:   0.011459
+Median  KLD:   0.002024
+10.0%   KLD:   0.000093
+ 5.0%   KLD:   0.000014
+ 1.0%   KLD:  -0.000000
+ 0.1%   KLD:  -0.000027
+Minimum KLD:  -0.000139
+
+====== Token probability statistics ======
+Mean    Δp: -0.129 ± 0.009 %
+Maximum Δp: 26.548%
+99.9%   Δp: 12.262%
+99.0%   Δp:  4.915%
+95.0%   Δp:  1.852%
+90.0%   Δp:  0.953%
+75.0%   Δp:  0.123%
+Median  Δp: -0.001%
+25.0%   Δp: -0.252%
+10.0%   Δp: -1.292%
+ 5.0%   Δp: -2.469%
+ 1.0%   Δp: -6.287%
+ 0.1%   Δp: -15.758%
+Minimum Δp: -56.803%
+RMS Δp    :  1.860 ± 0.038 %
+Same top p: 96.381 ± 0.088 %
+

scripts/run_amalia_pt.sh

@@ -0,0 +1,88 @@
+#!/bin/bash
+# Salamandra-7b-instruct full-ladder quant + KLD, calibrated on PORTUGUESE (the AMALIA lesson).
+set -o pipefail
+CONVERT=/Users/kikocisneros/llama.cpp/convert_hf_to_gguf.py
+QUANT=/opt/homebrew/bin/llama-quantize
+IMAT=/opt/homebrew/bin/llama-imatrix
+GGUF=/opt/homebrew/bin/llama-gguf
+PPX=/opt/homebrew/bin/llama-perplexity
+PY=/Users/kikocisneros/work2026/agentic-install/lab/gemma-e2b-cli/.venv/bin/python3
+GL=~/.gpu_coordination/gpu-lock.sh
+W=/Users/kikocisneros/coco_ppl/aq_amalia
+HF_DIR=$W/hf
+F16=$W/amalia-9b-f16.gguf
+IMATRIX=$W/amalia-9b-es.imatrix
+CORPUS=$W/pt_corpus.txt          # PORTUGUESE — for imatrix AND KLD eval
+KBASE=$W/kld_base.dat
+REP=$W/reports; mkdir -p "$HF_DIR" "$REP" "$W/metrics"
+LOG=$W/pipeline.log
+exec > >(tee -a "$LOG") 2>&1
+echo "==================== [$(date)] AMALIA PIPELINE START ===================="
+fail(){ echo "!!! FAIL: $1"; bash $GL release agentic-install 2>/dev/null; exit 1; }
+
+# 1. DOWNLOAD
+if [ ! -f "$HF_DIR/config.json" ]; then
+  echo "--- [1] download ---"
+  $PY - <<PY2 || fail "download"
+from huggingface_hub import snapshot_download
+snapshot_download("amalia-llm/AMALIA-9B-0626-SFT", local_dir="$HF_DIR",
+  allow_patterns=["*.safetensors","*.json","tokenizer*","*.txt","*.model","*.jinja"], max_workers=4)
+print("DL_DONE")
+PY2
+else echo "--- [1] download present ---"; fi
+
+# 2. CONVERT -> F16  (magic-check gates incomplete converts; no fragile load-test)
+if [ ! -f "$F16" ]; then
+  echo "--- [2] convert -> F16 ---"
+  $PY "$CONVERT" "$HF_DIR" --outfile "$F16" --outtype f16 || fail "convert"
+else echo "--- [2] F16 present ---"; fi
+"$GGUF" "$F16" r 2>&1 | grep -qi "invalid magic" && fail "F16 invalid magic"
+echo "--- [3] load-test SKIPPED (magic verified) ---"
+echo "F16 size: $(du -h "$F16" | cut -f1)"
+
+# 4. IMATRIX on PORTUGUESE (GPU)
+if [ ! -f "$IMATRIX" ]; then
+  echo "--- [4] imatrix (PORTUGUESE corpus, GPU) ---"
+  bash $GL acquire agentic-install high "AMALIA PT imatrix" 2>&1 | tail -1
+  "$IMAT" -m "$F16" -f "$CORPUS" -o "$IMATRIX" -c 512 -ngl 99 2>&1 | tail -4 || \
+    { echo "imatrix -ngl 99 fail, retry -ngl 0"; "$IMAT" -m "$F16" -f "$CORPUS" -o "$IMATRIX" -c 512 -ngl 0 2>&1 | tail -4; }
+  bash $GL release agentic-install 2>&1 | tail -1
+  [ -f "$IMATRIX" ] || fail "imatrix not produced"
+else echo "--- [4] imatrix present ---"; fi
+
+# 5. FULL LADDER
+echo "--- [5] full ladder ---"
+for Q in Q3_K_M IQ4_XS Q4_K_M Q5_K_M Q6_K Q8_0; do
+  OUTF="$W/amalia-9b-${Q}.gguf"; [ -f "$OUTF" ] && { echo "  $Q present"; continue; }
+  IMFLAG=""; [ "$Q" != "Q8_0" ] && IMFLAG="--imatrix $IMATRIX"
+  "$QUANT" $IMFLAG "$F16" "$OUTF" "$Q" > "$W/q_${Q}.log" 2>&1 || { echo "  $Q FAILED"; continue; }
+  "$GGUF" "$OUTF" r 2>&1 | grep -qi "invalid magic" && { echo "  $Q INVALID MAGIC"; rm -f "$OUTF"; } || echo "  $Q OK: $(du -h "$OUTF" | cut -f1)"
+done
+
+# 6. KLD SWEEP on PORTUGUESE
+echo "--- [6] KLD sweep (PORTUGUESE) ---"
+bash $GL acquire agentic-install high "AMALIA PT KLD" 2>&1 | tail -1
+"$PPX" -m "$F16" -f "$CORPUS" -c 512 --kl-divergence-base "$KBASE" -ngl 99 > "$REP/base.txt" 2>&1
+F16PPL=$(grep "Final estimate: PPL" "$REP/base.txt" | grep -oE "[0-9]+\.[0-9]+" | head -1)
+CSV=$W/metrics/quant-summary.csv
+echo "name,size_gb,ppl,ppl_delta_vs_f16,kld_mean,kld_p95,kld_max,top1_match_pct" > "$CSV"
+echo "F16,$(echo "scale=2;$(du -m "$F16"|cut -f1)/1024"|bc),${F16PPL},0.0,0.0,0.0,0.0,100.0" >> "$CSV"
+for Q in Q8_0 Q6_K Q5_K_M Q4_K_M IQ4_XS Q3_K_M; do
+  M=$W/amalia-9b-${Q}.gguf; [ -f "$M" ] || continue
+  R=$REP/kld-${Q}.txt
+  "$PPX" -m "$M" -f "$CORPUS" -c 512 --kl-divergence-base "$KBASE" --kl-divergence -ngl 99 > "$R" 2>&1
+  PPL=$(grep "Mean PPL(Q) " "$R" | grep -oE "[0-9]+\.[0-9]+" | head -1)
+  DELTA=$(grep "Mean PPL(Q)-PPL(base)" "$R" | grep -oE "\-?[0-9]+\.[0-9]+" | head -1)
+  KM=$(grep "Mean    KLD:" "$R" | grep -oE "[0-9]+\.[0-9]+" | head -1)
+  KP95=$(grep "95.0%   KLD:" "$R" | sed 's/.*KLD://' | grep -oE "[0-9]+\.[0-9]+" | head -1)
+  KMAX=$(grep "Maximum KLD:" "$R" | grep -oE "[0-9]+\.[0-9]+" | head -1)
+  T1=$(grep "Same top p:" "$R" | grep -oE "[0-9]+\.[0-9]+" | head -1)
+  SZ=$(echo "scale=2;$(du -m "$M"|cut -f1)/1024"|bc)
+  echo "  $Q: ${SZ}GB ppl=${PPL} dppl=${DELTA} kld=${KM} p95=${KP95} top1=${T1}%"
+  echo "${Q},${SZ},${PPL},${DELTA},${KM},${KP95},${KMAX},${T1}" >> "$CSV"
+done
+bash $GL release agentic-install 2>&1 | tail -1
+echo "==================== [$(date)] DONE ===================="
+echo "Produced:"; /bin/ls -lS "$W"/amalia-9b-*.gguf 2>/dev/null | awk '{print "  "$5"  "$9}'
+echo "--- metrics ---"; cat "$CSV"
+echo "AMALIA_COMPLETE"

scripts/run_kld2048.sh

@@ -0,0 +1,52 @@
+#!/bin/bash
+# AMALIA-9B — cleaner KLD re-eval at ctx 2048 (lower per-token surprise -> cleaner, monotonic KLD).
+# Reuses the existing F16 + ladder + imatrix; only the KLD eval re-runs. Also a coherence sanity gen.
+set -o pipefail
+QUANT=/opt/homebrew/bin/llama-quantize
+GGUF=/opt/homebrew/bin/llama-gguf
+PPX=/opt/homebrew/bin/llama-perplexity
+CLI=/opt/homebrew/bin/llama-cli
+GL=~/.gpu_coordination/gpu-lock.sh
+W=/Users/kikocisneros/coco_ppl/aq_amalia
+F16=$W/amalia-9b-f16.gguf
+CORPUS=$W/pt_corpus.txt
+KBASE=$W/kld_base_2048.dat
+CSV=$W/metrics/quant-summary.csv         # overwrite with the clean ctx-2048 numbers
+REP=$W/reports; mkdir -p "$REP" "$W/metrics"
+LOG=$W/kld2048.log
+exec > >(tee -a "$LOG") 2>&1
+echo "==================== [$(date)] AMALIA KLD ctx2048 START ===================="
+ppl_of(){ grep "Final estimate: PPL" "$1" | sed -E 's/.*PPL = ([0-9.]+).*/\1/' | head -1; }
+
+bash $GL acquire agentic-install high "AMALIA KLD ctx2048" 2>&1 | tail -1
+
+# (coherence check removed — llama-cli hangs on stdin; Q8's tiny KLD already proves F16 is sane)
+
+# 1. F16 base at ctx 2048
+echo "--- [1] F16 base (ctx 2048) ---"
+"$PPX" -m "$F16" -f "$CORPUS" -c 2048 --kl-divergence-base "$KBASE" -ngl 99 > "$REP/base2048.txt" 2>&1 || { echo "base FAIL"; bash $GL release agentic-install; exit 1; }
+F16PPL=$(ppl_of "$REP/base2048.txt")
+echo "  F16 PPL(ctx2048) = ${F16PPL}"
+echo "name,size_gb,ppl,ppl_delta_vs_f16,kld_mean,kld_p95,kld_max,top1_match_pct" > "$CSV"
+echo "F16,17.05,${F16PPL},0.0,0.0,0.0,0.0,100.0" >> "$CSV"
+
+# 2. Each quant vs base at ctx 2048
+for Q in Q8_0 Q6_K Q5_K_M Q4_K_M IQ4_XS Q3_K_M; do
+  M=$W/amalia-9b-${Q}.gguf; [ -f "$M" ] || { echo "  $Q missing"; continue; }
+  R=$REP/kld2048-${Q}.txt
+  echo "--- [2] $Q (ctx 2048) ---"
+  "$PPX" -m "$M" -f "$CORPUS" -c 2048 --kl-divergence-base "$KBASE" --kl-divergence -ngl 99 > "$R" 2>&1
+  PPL=$(grep "Mean PPL(Q) " "$R"|grep -oE "[0-9]+\.[0-9]+"|head -1)
+  DELTA=$(grep "Mean PPL(Q)-PPL(base)" "$R"|grep -oE "\-?[0-9]+\.[0-9]+"|head -1)
+  KM=$(grep "Mean    KLD:" "$R"|grep -oE "[0-9]+\.[0-9]+"|head -1)
+  KP95=$(grep "95.0%   KLD:" "$R"|sed 's/.*KLD://'|grep -oE "[0-9]+\.[0-9]+"|head -1)
+  KMAX=$(grep "Maximum KLD:" "$R"|grep -oE "[0-9]+\.[0-9]+"|head -1)
+  T1=$(grep "Same top p:" "$R"|grep -oE "[0-9]+\.[0-9]+"|head -1)
+  SZ=$(du -m "$M"|awk '{printf "%.2f",$1/1024}')
+  echo "  $Q: ${SZ}GB ppl=${PPL} dppl=${DELTA} kld=${KM} p95=${KP95} top1=${T1}%"
+  echo "${Q},${SZ},${PPL},${DELTA},${KM},${KP95},${KMAX},${T1}" >> "$CSV"
+done
+bash $GL release agentic-install 2>&1 | tail -1
+echo "==================== [$(date)] DONE ===================="
+cat "$CSV"
+echo "KLD2048_COMPLETE"