Upload scripts/v26_pq/publish/make_charts.py with huggingface_hub

scripts/v26_pq/publish/make_charts.py

@@ -0,0 +1,94 @@
+#!/usr/bin/env python3
+"""Generate charts for the v26 HF model card."""
+import json
+import sys
+from pathlib import Path
+
+import matplotlib
+matplotlib.use("Agg")
+import matplotlib.pyplot as plt
+
+HERE = Path("/home/jaime/lut_work/dna-2b-tests/v26_pq/publish")
+OUT = HERE / "charts"
+OUT.mkdir(parents=True, exist_ok=True)
+REPORT = "/home/jaime/lut_work/dna-2b-tests/v25_eval/results/report.json"
+GATE_V26 = "/home/jaime/lut_work/dna-2b-tests/v26_pq/results/gate_v26.json"
+GATE_V25 = "/home/jaime/lut_work/dna-2b-tests/v26_pq/results/gate_g8b12.json"
+
+BLUE = "#2b6cb0"
+RED = "#c53030"
+GREEN = "#2f855a"
+GRAY = "#718096"
+
+# 1) Training throughput (v25 base, same arch as v26)
+r = json.load(open(REPORT))
+series = [s for s in r["training"]["series"] if "v25-base" in s.get("source", "")]
+steps = [s["step"] for s in series]
+tps = [s["tokens_per_second"] for s in series]
+fig, ax = plt.subplots(figsize=(8, 4.5), dpi=130)
+ax.plot(steps, tps, color=BLUE, lw=1.4)
+ax.axhline(r["training"]["mean_tokens_per_second"], color=GREEN, ls="--", lw=1,
+           label=f"mean {r['training']['mean_tokens_per_second']:.0f} tok/s")
+ax.set_xlabel("step")
+ax.set_ylabel("tokens/s")
+ax.set_title("v25/v26 base pretraining throughput (same arch)")
+ax.legend()
+ax.grid(alpha=0.25)
+fig.tight_layout(); fig.savefig(OUT / "training_tok_s.png"); plt.close(fig)
+
+# 2) Compression CE delta: v25 ternary vs v26 all-fp16 vs v26 mixed
+g25 = json.load(open(GATE_V25))
+g26 = json.load(open(GATE_V26))
+labels = ["v25 ternary\n(+4.95)", "v26 all-fp16\n(+0.026)", "v26 mixed\n(-0.028)"]
+deltas = [4.952, 0.0255, -0.0276]
+colors = [RED, GRAY, GREEN]
+fig, ax = plt.subplots(figsize=(7, 4.5), dpi=130)
+bars = ax.bar(labels, deltas, color=colors)
+ax.axhline(0.01, color=RED, ls="--", lw=1, label="quality gate +0.01")
+ax.axhline(0, color="black", lw=0.8)
+for b, d in zip(bars, deltas):
+    ax.text(b.get_x() + b.get_width() / 2, b.get_height() + (0.12 if d >= 0 else -0.35),
+            f"{d:+.3f}", ha="center", fontsize=11, fontweight="bold")
+ax.set_ylabel("compression validation CE delta")
+ax.set_title("Compression quality: v25 vs v26 codecs")
+ax.legend()
+ax.grid(axis="y", alpha=0.25)
+fig.tight_layout(); fig.savefig(OUT / "compression_ce_delta.png"); plt.close(fig)
+
+# 3) Size comparison
+labels = ["v25 ternary\n186.6 MiB", "v26 all-fp16\n166.6 MiB", "v26 mixed\n176.6 MiB", "195 MiB cap"]
+sizes = [186.6, 166.6, 176.6, 195.0]
+colors = [RED, GRAY, GREEN, "#00000022"]
+fig, ax = plt.subplots(figsize=(7, 4.5), dpi=130)
+bars = ax.bar(labels, sizes, color=colors)
+for b, s in zip(bars, sizes):
+    ax.text(b.get_x() + b.get_width() / 2, s + 2, f"{s:.1f}", ha="center", fontsize=11, fontweight="bold")
+ax.set_ylabel("MiB")
+ax.set_title("Deployment checkpoint size (2B logical params)")
+ax.grid(axis="y", alpha=0.25)
+fig.tight_layout(); fig.savefig(OUT / "checkpoint_size.png"); plt.close(fig)
+
+# 4) HumanEval comparison vs peers (web-sourced, brave searches logged)
+models = ["Qwen2.5-Coder\n1.5B", "SmolLM2\n1.7B", "Gemma-2\n2B", "TinyLlama\n1.1B",
+          "Phi-2\n2.7B", "DNA-DiskChat\n2B PEER v26"]
+he = [43.3, 28.1, 17.7, 8.5, 59.2, 0.0]
+mbpp = [50.0, None, None, 12.3, 59.1, 0.0]
+fig, ax = plt.subplots(figsize=(8.5, 4.8), dpi=130)
+x = range(len(models))
+b1 = ax.bar([i - 0.2 for i in x], he, width=0.4, color=BLUE, label="HumanEval pass@1")
+for i, v in enumerate(he):
+    ax.text(i - 0.2, v + 1.2, f"{v:.1f}", ha="center", fontsize=9)
+for i, v in enumerate(mbpp):
+    if v is not None:
+        ax.bar(i + 0.2, v, width=0.4, color=GREEN, label="MBPP pass@1" if i == 0 else None)
+        ax.text(i + 0.2, v + 1.2, f"{v:.1f}", ha="center", fontsize=9)
+ax.axhline(0, color="black", lw=0.8)
+ax.set_xticks(list(x)); ax.set_xticklabels(models, fontsize=9)
+ax.set_ylabel("pass@1 (%)")
+ax.set_title("HumanEval / MBPP: DNA-DiskChat-2B-PEER-v26 vs same-size peers")
+ax.legend()
+ax.grid(axis="y", alpha=0.25)
+ax.annotate("v26: 0% — honest result, see README", xy=(7, 0.5), fontsize=10, color=RED, ha="right")
+fig.tight_layout(); fig.savefig(OUT / "benchmark_comparison.png"); plt.close(fig)
+
+print("charts written:", sorted(p.name for p in OUT.iterdir()))