Upload gemma-27b_layer58_ml50-54-58_maghead_mlp_h2048x2_t0.05_a1.0.metrics.json with huggingface_hub
Browse files
gemma-27b_layer58_ml50-54-58_maghead_mlp_h2048x2_t0.05_a1.0.metrics.json
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"checkpoint": "gemma-27b_layer58_ml50-54-58_maghead_mlp_h2048x2_t0.05_a1.0.pt",
|
| 3 |
+
"arch": "maghead_mlp, MULTI-LAYER input (concat residuals of layers 50,54,58 -> 16128-d) + linear mag head",
|
| 4 |
+
"base_model": "gemma-3-27b",
|
| 5 |
+
"data": "fineweb-gemma27b-residuals (24 shards)",
|
| 6 |
+
"layers": [
|
| 7 |
+
50,
|
| 8 |
+
54,
|
| 9 |
+
58
|
| 10 |
+
],
|
| 11 |
+
"d_res": 16128,
|
| 12 |
+
"data_shards": 24,
|
| 13 |
+
"val_rank1": 0.676,
|
| 14 |
+
"val_dir_rank1": 0.693,
|
| 15 |
+
"val_cos": 0.443,
|
| 16 |
+
"val_mag_mse": 0.685,
|
| 17 |
+
"date": "2026-06-07",
|
| 18 |
+
"note": "OVERALL RETRIEVAL SOTA across all parascope probes. Multi-layer input (3 mid+deep layers) + 24sh data stack: single-L58-16sh 0.658 -> ml3-12sh 0.661 -> ml3-24sh 0.676. dir_rank1 0.693. Decode still magnitude-capped (mag_mse ~0.69); multi-layer lifts direction/retrieval, not the magnitude wall."
|
| 19 |
+
}
|