{ "checkpoint_config": { "activation": "gelu", "actual_precision": "bf16", "adamw_betas": [ 0.9, 0.95 ], "adamw_eps": 1e-08, "architecture": "gpt2", "attention_kernel_policy": "auto", "batch_size": 2, "benchmark": { "enable_central_tensorboard": true, "enable_local_tensorboard": true, "enabled": false, "output_path": "/mnt/apps/llm-nanochat/artifacts/runs/20260629_resume-gpt2medium-gpt2preln-k20-wsddecayonly-rerunmissing-lr3p5294e5-anchor20k-final2e5-webwiki-step14200-to14850/throughput_benchmark.json", "warmup_steps": 0 }, "bias": true, "block_type": "gpt2_prelayernorm", "causal_mask_mode": "buffered_upper_triangular", "checkpoint_dir": "/mnt/apps/llm-nanochat/checkpoints/20260629_resume-gpt2medium-gpt2preln-k20-wsddecayonly-rerunmissing-lr3p5294e5-anchor20k-final2e5-webwiki-step14200-to14850", "clip_grad_norm": 1.0, "compile": { "backend": null, "compile_setup_sec": 0.0, "diagnostic": null, "dynamic": false, "enabled": false, "error_policy": "raise", "fullgraph": false, "mode": null, "requested": false, "status": "disabled" }, "dataset": { "storage_mode": "indexed_jsonl" }, "decay_shape": "inverse_proportional", "decay_steps": 650, "decay_steps_source": "explicit", "deterministic_algorithms": false, "device": "cuda", "dim": 1024, "dropout": 0.0, "effective_layer_count": 24, "final_lr": 2e-05, "final_lr_source": "explicit", "fp8_backend": null, "grad_accum_steps": 48, "init_strategy": "gpt2_std_0.02_residual_scale", "learning_rate": 3.529411764705883e-05, "logging": { "enable_central_tensorboard": true, "enable_local_tensorboard": true, "metrics_flush_every_steps": 1, "metrics_writer": "persistent_jsonl_handle" }, "lr": 3.529411764705883e-05, "lr_schedule": "wsd-decay-only", "max_seq_len": 2500, "max_steps": 14850, "n_heads": 16, "n_layers": 24, "norm_order": "preln", "norm_type": "layernorm", "optimizer": { "backend": "torch", "betas": [ 0.9, 0.95 ], "eps": 1e-08, "implementation": "torch.optim.AdamW", "learning_rate": 3.529411764705883e-05, "state_precision": "full_precision", "type": "adamw", "weight_decay": 0.1 }, "optimizer_backend": "torch", "optimizer_implementation": "torch.optim.AdamW", "optimizer_state_precision": "full_precision", "optimizer_type": "adamw", "peak_lr": 3.529411764705883e-05, "positional_encoding": "learned_absolute", "repro": { "attention_kernel_policy": "auto", "cublas_workspace_config": null, "cudnn_benchmark": true, "cudnn_deterministic": false, "deterministic_algorithms": false, "flash_sdp_enabled": true, "math_sdp_enabled": true, "mem_efficient_sdp_enabled": true, "pythonhashseed": "1337", "seed": 1337 }, "requested_precision": "bf16", "resume_from": "/mnt/dati/llm-nanochat/checkpoints_archive/tier2_20260629_medium_prune/20260628_resume-gpt2medium-gpt2preln-k20-wsddecayonly-lr2e-4-anchor20k-final2e5-webwiki-step13500/step_14200.pt", "resume_mode": "model_only", "resume_step": 14200, "save_every_steps": 50, "scheduler": { "decay_shape": "inverse_proportional", "decay_steps": 650, "decay_steps_source": "explicit", "final_lr": 2e-05, "final_lr_source": "explicit", "peak_lr": 3.529411764705883e-05, "resume_step": 14200, "schedule_type": "wsd-decay-only", "stable_steps": 0, "target_final_step": 14850, "total_steps": 14850, "warmup_steps": 0 }, "seed": 1337, "stable_steps": 0, "target_final_step": 14850, "tie_word_embeddings": true, "train_cache_ram_bytes": 1073741824, "train_cache_ram_mb": 1024, "vocab_size": 32000, "warmup_steps": 0, "weight_decay": 0.1 }, "checkpoint_path": "/mnt/apps/llm-nanochat/checkpoints/20260629_resume-gpt2medium-gpt2preln-k20-wsddecayonly-rerunmissing-lr3p5294e5-anchor20k-final2e5-webwiki-step14200-to14850/step_14700.pt", "config_path": "/mnt/apps/llm-nanochat/hf_exports/gpt2medium-en-it-nanochat-gpt2preln-lr2e4-bs2-ga48-wsddecayonly-anchor20k-final2e5-webwiki-step14700/config.json", "exported_at": "2026-06-29T22:36:28.065785+00:00", "format": "llm-nanochat-safetensors-export", "global_step": 14700, "metadata_path": "/mnt/apps/llm-nanochat/hf_exports/gpt2medium-en-it-nanochat-gpt2preln-lr2e4-bs2-ga48-wsddecayonly-anchor20k-final2e5-webwiki-step14700/step_14700.safetensors.json", "model_config": { "activation": "gelu", "architecture": "gpt2", "bias": true, "block_type": "gpt2_prelayernorm", "causal_mask_mode": "buffered_upper_triangular", "dim": 1024, "dropout": 0.0, "init_strategy": "gpt2_std_0.02_residual_scale", "max_seq_len": 2500, "n_heads": 16, "n_layers": 24, "norm_order": "preln", "norm_type": "layernorm", "positional_encoding": "learned_absolute", "tie_word_embeddings": true, "vocab_size": 32000 }, "num_parameters": 370439424, "num_tensors": 294, "provenance": { "checkpoint_dir": "/mnt/apps/llm-nanochat/checkpoints/20260629_resume-gpt2medium-gpt2preln-k20-wsddecayonly-rerunmissing-lr3p5294e5-anchor20k-final2e5-webwiki-step14200-to14850", "checkpoint_name": "step_14700.pt", "checkpoint_path": "/mnt/apps/llm-nanochat/checkpoints/20260629_resume-gpt2medium-gpt2preln-k20-wsddecayonly-rerunmissing-lr3p5294e5-anchor20k-final2e5-webwiki-step14200-to14850/step_14700.pt", "global_step": 14700, "packed_dataset_config_path": null, "run_dir": "/mnt/apps/llm-nanochat/checkpoints", "tokenizer_dir": "/home/descanso/.openclaw/workspace/python_project/llm-nanochat/artifacts/tokenizer_en_it_real_pilot_m_32k", "training_config_path": null }, "safetensors_path": "/mnt/apps/llm-nanochat/hf_exports/gpt2medium-en-it-nanochat-gpt2preln-lr2e4-bs2-ga48-wsddecayonly-anchor20k-final2e5-webwiki-step14700/step_14700.safetensors", "source_checkpoint_path": "/mnt/apps/llm-nanochat/checkpoints/20260629_resume-gpt2medium-gpt2preln-k20-wsddecayonly-rerunmissing-lr3p5294e5-anchor20k-final2e5-webwiki-step14200-to14850/step_14700.pt", "source_global_step": 14700, "tensor_names": [ "token_emb.weight", "pos_emb.weight", "blocks.0.ln_1.weight", "blocks.0.ln_1.bias", "blocks.0.ln_2.weight", "blocks.0.ln_2.bias", "blocks.0.attn.qkv.weight", "blocks.0.attn.qkv.bias", "blocks.0.attn.out_proj.weight", "blocks.0.attn.out_proj.bias", "blocks.0.mlp.fc.weight", "blocks.0.mlp.fc.bias", "blocks.0.mlp.proj.weight", "blocks.0.mlp.proj.bias", "blocks.1.ln_1.weight", "blocks.1.ln_1.bias", "blocks.1.ln_2.weight", "blocks.1.ln_2.bias", "blocks.1.attn.qkv.weight", "blocks.1.attn.qkv.bias", "blocks.1.attn.out_proj.weight", "blocks.1.attn.out_proj.bias", "blocks.1.mlp.fc.weight", "blocks.1.mlp.fc.bias", "blocks.1.mlp.proj.weight", "blocks.1.mlp.proj.bias", "blocks.2.ln_1.weight", "blocks.2.ln_1.bias", "blocks.2.ln_2.weight", "blocks.2.ln_2.bias", "blocks.2.attn.qkv.weight", "blocks.2.attn.qkv.bias", "blocks.2.attn.out_proj.weight", "blocks.2.attn.out_proj.bias", "blocks.2.mlp.fc.weight", "blocks.2.mlp.fc.bias", "blocks.2.mlp.proj.weight", "blocks.2.mlp.proj.bias", "blocks.3.ln_1.weight", "blocks.3.ln_1.bias", "blocks.3.ln_2.weight", "blocks.3.ln_2.bias", "blocks.3.attn.qkv.weight", "blocks.3.attn.qkv.bias", "blocks.3.attn.out_proj.weight", "blocks.3.attn.out_proj.bias", "blocks.3.mlp.fc.weight", "blocks.3.mlp.fc.bias", "blocks.3.mlp.proj.weight", "blocks.3.mlp.proj.bias", "blocks.4.ln_1.weight", "blocks.4.ln_1.bias", "blocks.4.ln_2.weight", "blocks.4.ln_2.bias", "blocks.4.attn.qkv.weight", "blocks.4.attn.qkv.bias", "blocks.4.attn.out_proj.weight", "blocks.4.attn.out_proj.bias", "blocks.4.mlp.fc.weight", "blocks.4.mlp.fc.bias", "blocks.4.mlp.proj.weight", "blocks.4.mlp.proj.bias", "blocks.5.ln_1.weight", "blocks.5.ln_1.bias", "blocks.5.ln_2.weight", "blocks.5.ln_2.bias", "blocks.5.attn.qkv.weight", "blocks.5.attn.qkv.bias", "blocks.5.attn.out_proj.weight", "blocks.5.attn.out_proj.bias", "blocks.5.mlp.fc.weight", "blocks.5.mlp.fc.bias", "blocks.5.mlp.proj.weight", "blocks.5.mlp.proj.bias", "blocks.6.ln_1.weight", "blocks.6.ln_1.bias", "blocks.6.ln_2.weight", "blocks.6.ln_2.bias", "blocks.6.attn.qkv.weight", "blocks.6.attn.qkv.bias", "blocks.6.attn.out_proj.weight", "blocks.6.attn.out_proj.bias", "blocks.6.mlp.fc.weight", "blocks.6.mlp.fc.bias", "blocks.6.mlp.proj.weight", "blocks.6.mlp.proj.bias", "blocks.7.ln_1.weight", "blocks.7.ln_1.bias", "blocks.7.ln_2.weight", "blocks.7.ln_2.bias", "blocks.7.attn.qkv.weight", "blocks.7.attn.qkv.bias", "blocks.7.attn.out_proj.weight", "blocks.7.attn.out_proj.bias", "blocks.7.mlp.fc.weight", "blocks.7.mlp.fc.bias", "blocks.7.mlp.proj.weight", "blocks.7.mlp.proj.bias", "blocks.8.ln_1.weight", "blocks.8.ln_1.bias", "blocks.8.ln_2.weight", "blocks.8.ln_2.bias", "blocks.8.attn.qkv.weight", "blocks.8.attn.qkv.bias", "blocks.8.attn.out_proj.weight", "blocks.8.attn.out_proj.bias", "blocks.8.mlp.fc.weight", "blocks.8.mlp.fc.bias", "blocks.8.mlp.proj.weight", "blocks.8.mlp.proj.bias", "blocks.9.ln_1.weight", "blocks.9.ln_1.bias", "blocks.9.ln_2.weight", "blocks.9.ln_2.bias", "blocks.9.attn.qkv.weight", "blocks.9.attn.qkv.bias", "blocks.9.attn.out_proj.weight", "blocks.9.attn.out_proj.bias", "blocks.9.mlp.fc.weight", "blocks.9.mlp.fc.bias", "blocks.9.mlp.proj.weight", "blocks.9.mlp.proj.bias", "blocks.10.ln_1.weight", "blocks.10.ln_1.bias", "blocks.10.ln_2.weight", "blocks.10.ln_2.bias", "blocks.10.attn.qkv.weight", "blocks.10.attn.qkv.bias", "blocks.10.attn.out_proj.weight", "blocks.10.attn.out_proj.bias", "blocks.10.mlp.fc.weight", "blocks.10.mlp.fc.bias", "blocks.10.mlp.proj.weight", "blocks.10.mlp.proj.bias", "blocks.11.ln_1.weight", "blocks.11.ln_1.bias", "blocks.11.ln_2.weight", "blocks.11.ln_2.bias", "blocks.11.attn.qkv.weight", "blocks.11.attn.qkv.bias", "blocks.11.attn.out_proj.weight", "blocks.11.attn.out_proj.bias", "blocks.11.mlp.fc.weight", "blocks.11.mlp.fc.bias", "blocks.11.mlp.proj.weight", "blocks.11.mlp.proj.bias", "blocks.12.ln_1.weight", "blocks.12.ln_1.bias", "blocks.12.ln_2.weight", "blocks.12.ln_2.bias", "blocks.12.attn.qkv.weight", "blocks.12.attn.qkv.bias", "blocks.12.attn.out_proj.weight", "blocks.12.attn.out_proj.bias", "blocks.12.mlp.fc.weight", "blocks.12.mlp.fc.bias", "blocks.12.mlp.proj.weight", "blocks.12.mlp.proj.bias", "blocks.13.ln_1.weight", "blocks.13.ln_1.bias", "blocks.13.ln_2.weight", "blocks.13.ln_2.bias", "blocks.13.attn.qkv.weight", "blocks.13.attn.qkv.bias", "blocks.13.attn.out_proj.weight", "blocks.13.attn.out_proj.bias", "blocks.13.mlp.fc.weight", "blocks.13.mlp.fc.bias", "blocks.13.mlp.proj.weight", "blocks.13.mlp.proj.bias", "blocks.14.ln_1.weight", "blocks.14.ln_1.bias", "blocks.14.ln_2.weight", "blocks.14.ln_2.bias", "blocks.14.attn.qkv.weight", "blocks.14.attn.qkv.bias", "blocks.14.attn.out_proj.weight", "blocks.14.attn.out_proj.bias", "blocks.14.mlp.fc.weight", "blocks.14.mlp.fc.bias", "blocks.14.mlp.proj.weight", "blocks.14.mlp.proj.bias", "blocks.15.ln_1.weight", "blocks.15.ln_1.bias", "blocks.15.ln_2.weight", "blocks.15.ln_2.bias", "blocks.15.attn.qkv.weight", "blocks.15.attn.qkv.bias", "blocks.15.attn.out_proj.weight", "blocks.15.attn.out_proj.bias", "blocks.15.mlp.fc.weight", "blocks.15.mlp.fc.bias", "blocks.15.mlp.proj.weight", "blocks.15.mlp.proj.bias", "blocks.16.ln_1.weight", "blocks.16.ln_1.bias", "blocks.16.ln_2.weight", "blocks.16.ln_2.bias", "blocks.16.attn.qkv.weight", "blocks.16.attn.qkv.bias", "blocks.16.attn.out_proj.weight", "blocks.16.attn.out_proj.bias", "blocks.16.mlp.fc.weight", "blocks.16.mlp.fc.bias", "blocks.16.mlp.proj.weight", "blocks.16.mlp.proj.bias", "blocks.17.ln_1.weight", "blocks.17.ln_1.bias", "blocks.17.ln_2.weight", "blocks.17.ln_2.bias", "blocks.17.attn.qkv.weight", "blocks.17.attn.qkv.bias", "blocks.17.attn.out_proj.weight", "blocks.17.attn.out_proj.bias", "blocks.17.mlp.fc.weight", "blocks.17.mlp.fc.bias", "blocks.17.mlp.proj.weight", "blocks.17.mlp.proj.bias", "blocks.18.ln_1.weight", "blocks.18.ln_1.bias", "blocks.18.ln_2.weight", "blocks.18.ln_2.bias", "blocks.18.attn.qkv.weight", "blocks.18.attn.qkv.bias", "blocks.18.attn.out_proj.weight", "blocks.18.attn.out_proj.bias", "blocks.18.mlp.fc.weight", "blocks.18.mlp.fc.bias", "blocks.18.mlp.proj.weight", "blocks.18.mlp.proj.bias", "blocks.19.ln_1.weight", "blocks.19.ln_1.bias", "blocks.19.ln_2.weight", "blocks.19.ln_2.bias", "blocks.19.attn.qkv.weight", "blocks.19.attn.qkv.bias", "blocks.19.attn.out_proj.weight", "blocks.19.attn.out_proj.bias", "blocks.19.mlp.fc.weight", "blocks.19.mlp.fc.bias", "blocks.19.mlp.proj.weight", "blocks.19.mlp.proj.bias", "blocks.20.ln_1.weight", "blocks.20.ln_1.bias", "blocks.20.ln_2.weight", "blocks.20.ln_2.bias", "blocks.20.attn.qkv.weight", "blocks.20.attn.qkv.bias", "blocks.20.attn.out_proj.weight", "blocks.20.attn.out_proj.bias", "blocks.20.mlp.fc.weight", "blocks.20.mlp.fc.bias", "blocks.20.mlp.proj.weight", "blocks.20.mlp.proj.bias", "blocks.21.ln_1.weight", "blocks.21.ln_1.bias", "blocks.21.ln_2.weight", "blocks.21.ln_2.bias", "blocks.21.attn.qkv.weight", "blocks.21.attn.qkv.bias", "blocks.21.attn.out_proj.weight", "blocks.21.attn.out_proj.bias", "blocks.21.mlp.fc.weight", "blocks.21.mlp.fc.bias", "blocks.21.mlp.proj.weight", "blocks.21.mlp.proj.bias", "blocks.22.ln_1.weight", "blocks.22.ln_1.bias", "blocks.22.ln_2.weight", "blocks.22.ln_2.bias", "blocks.22.attn.qkv.weight", "blocks.22.attn.qkv.bias", "blocks.22.attn.out_proj.weight", "blocks.22.attn.out_proj.bias", "blocks.22.mlp.fc.weight", "blocks.22.mlp.fc.bias", "blocks.22.mlp.proj.weight", "blocks.22.mlp.proj.bias", "blocks.23.ln_1.weight", "blocks.23.ln_1.bias", "blocks.23.ln_2.weight", "blocks.23.ln_2.bias", "blocks.23.attn.qkv.weight", "blocks.23.attn.qkv.bias", "blocks.23.attn.out_proj.weight", "blocks.23.attn.out_proj.bias", "blocks.23.mlp.fc.weight", "blocks.23.mlp.fc.bias", "blocks.23.mlp.proj.weight", "blocks.23.mlp.proj.bias", "ln_f.weight", "ln_f.bias", "head.weight", "head.bias" ], "tokenizer_bundle": { "special_tokens_map.json": "/mnt/apps/llm-nanochat/hf_exports/gpt2medium-en-it-nanochat-gpt2preln-lr2e4-bs2-ga48-wsddecayonly-anchor20k-final2e5-webwiki-step14700/special_tokens_map.json", "tokenizer.json": "/mnt/apps/llm-nanochat/hf_exports/gpt2medium-en-it-nanochat-gpt2preln-lr2e4-bs2-ga48-wsddecayonly-anchor20k-final2e5-webwiki-step14700/tokenizer.json", "tokenizer_config.json": "/mnt/apps/llm-nanochat/hf_exports/gpt2medium-en-it-nanochat-gpt2preln-lr2e4-bs2-ga48-wsddecayonly-anchor20k-final2e5-webwiki-step14700/tokenizer_config.json", "tokenizer_meta.json": "/mnt/apps/llm-nanochat/hf_exports/gpt2medium-en-it-nanochat-gpt2preln-lr2e4-bs2-ga48-wsddecayonly-anchor20k-final2e5-webwiki-step14700/tokenizer_meta.json" }, "tokenizer_reference": { "packed_dataset_config_path": null, "tokenizer_dir": "/home/descanso/.openclaw/workspace/python_project/llm-nanochat/artifacts/tokenizer_en_it_real_pilot_m_32k", "training_config_path": null }, "transformers_config": { "activation_function": "gelu", "architecture": "gpt2", "architectures": [ "GPT2LMHeadModel" ], "attn_pdrop": 0.0, "block_type": "gpt2_prelayernorm", "causal_mask_mode": "buffered_upper_triangular", "embd_pdrop": 0.0, "init_strategy": "gpt2_std_0.02_residual_scale", "layer_norm_epsilon": 1e-05, "model_type": "gpt2", "n_ctx": 2500, "n_embd": 1024, "n_head": 16, "n_layer": 24, "n_positions": 2500, "norm_order": "preln", "norm_type": "layernorm", "positional_encoding": "learned_absolute", "resid_pdrop": 0.0, "tie_word_embeddings": true, "use_cache": true, "vocab_size": 32000 }, "transformers_export_notes": { "dropped_lm_head_bias_for_gpt2_compat": true, "dropped_lm_head_bias_max_abs": 0.8722193241119385, "dropped_lm_head_bias_mean_abs": 0.036528702825307846 }, "transformers_safetensors_path": "/mnt/apps/llm-nanochat/hf_exports/gpt2medium-en-it-nanochat-gpt2preln-lr2e4-bs2-ga48-wsddecayonly-anchor20k-final2e5-webwiki-step14700/model.safetensors" }