| { | |
| "model": "NeuroMamba-v5-NoFastWeight", | |
| "n_params": 28809768, | |
| "d_model": 312, | |
| "n_q": 6, | |
| "n_kv": 2, | |
| "d_head": 52, | |
| "d_ffn": 896, | |
| "window": 128, | |
| "n_local": 5, | |
| "n_global": 1, | |
| "n_cycles": 2, | |
| "seq_len": 512, | |
| "batch_size": 16, | |
| "grad_accum": 4, | |
| "lr": 0.0003, | |
| "warmup": 500, | |
| "target_tokens": 600000000, | |
| "tokens_seen": 599982080, | |
| "best_eval_loss": 4.442645943164825, | |
| "best_eval_perplexity": 84.99954836853307, | |
| "baseline_eval_loss": 4.3645, | |
| "delta_vs_baseline": 0.07814594316482548 | |
| } |