# Fresh GPT2-small web/wiki run with WSD short-fast decay to 11k.
# Goal: keep the useful high-LR early learning phase, but compress the stable+decay
# window into an 11k-step fresh run for a faster short-cycle benchmark.
# Schedule: warmup 500, stable 6500, decay 4000, final_lr 5e-6.
# No resume semantics: random weights, fresh optimizer, fresh scheduler.

dataset_dir: /mnt/apps/llm-nanochat/datasets/202605141153_fineweb50_wiki50_50en_50it_score100_2500context_5Btokens_tok_20260515_en50it50_webwiki_stratified_500M
output_dir: /mnt/apps/llm-nanochat/artifacts/runs/20260603_fresh-gpt2small-lr2e4-bs6-wsd-shortfastdecay11k-final5e6-webwiki
tokenizer_dir: /mnt/apps/llm-nanochat/tokenizers/tokenizer_20260515_en50it50_webwiki_stratified_500M
seed: 1337

model:
  vocab_size: 32000
  dim: 768
  n_layers: 12
  n_heads: 12

training:
  sequence_length: 2500
  max_steps: 11000
  batch_size: 6
  grad_accum_steps: 16

  learning_rate: 0.0002
  peak_lr: 0.0002
  lr_schedule: wsd

  warmup_steps: 500
  stable_steps: 6500
  decay_steps: 4000
  final_lr: 5.0e-06

  adamw_betas:
    - 0.9
    - 0.95
  adamw_eps: 1.0e-08
  weight_decay: 0.1
  clip_grad_norm: 1.0

  save_every_steps: 500
  checkpoint_dir: /mnt/apps/llm-nanochat/checkpoints/20260603_fresh-gpt2small-lr2e4-bs6-wsd-shortfastdecay11k-final5e6-webwiki
  precision: bf16

evaluation:
  validation_every_steps: 1000
  validation_max_batches: 128
  probe_every_steps: 1000
  probe_tokenizer_dir: /mnt/apps/llm-nanochat/tokenizers/tokenizer_20260515_en50it50_webwiki_stratified_500M
  probe_max_new_tokens: 32
  probe_prompts:
    en:
      - prompt: "The capital of Italy is"
        expected_next_text: " Rome"
      - prompt: "A small language model should"
        expected_next_text: " be"
    it:
      - prompt: "La capitale d'Italia è"
        expected_next_text: " Roma"
      - prompt: "Un piccolo modello linguistico dovrebbe"
        expected_next_text: " essere"