# WSD-decay-only continuation from the medium continual-pretraining checkpoint at step_22000.
# Intention: branch from a checkpoint that was still inside the stable plateau of the parent CPT run,
# then apply a shorter explicit decay-only cooldown for dense checkpoint comparison.

resume_from: /mnt/apps/llm-nanochat/checkpoints/20260703_continual-pretraining-gpt2medium-gpt2preln-k20-step14700-lr5e5-w500-s18500-d2000-final1e5-webwiki/step_22000.pt
dataset_dir: /mnt/apps/llm-nanochat/datasets/202605141153_fineweb50_wiki50_50en_50it_score100_2500context_5Btokens_tok_20260515_en50it50_webwiki_stratified_500M
output_dir: /mnt/apps/llm-nanochat/artifacts/runs/20260713_resume-gpt2medium-gpt2preln-k20-wsddecayonly-cpt14700-step22000-lr5e5-final1e5-webwiki-d1800
tokenizer_dir: /mnt/apps/llm-nanochat/tokenizers/tokenizer_20260515_en50it50_webwiki_stratified_500M
seed: 1337

model:
  architecture: gpt2
  block_type: gpt2_prelayernorm
  tie_word_embeddings: true
  vocab_size: 32000
  dim: 1024
  n_layers: 24
  n_heads: 16

training:
  sequence_length: 2500
  max_steps: 23800
  batch_size: 2
  grad_accum_steps: 48

  learning_rate: 5.0e-05
  peak_lr: 5.0e-05
  lr_schedule: wsd-decay-only

  warmup_steps: 0
  stable_steps: 0
  decay_steps: 1800
  final_lr: 1.0e-05

  adamw_betas:
    - 0.9
    - 0.95
  adamw_eps: 1.0e-08
  weight_decay: 0.1
  clip_grad_norm: 1.0

  save_every_steps: 100
  checkpoint_dir: /mnt/apps/llm-nanochat/checkpoints/20260713_resume-gpt2medium-gpt2preln-k20-wsddecayonly-cpt14700-step22000-lr5e5-final1e5-webwiki-d1800
  precision: bf16

evaluation:
  validation_every_steps: 100
  validation_max_batches: 128
  probe_every_steps: 1000
  probe_tokenizer_dir: /mnt/apps/llm-nanochat/tokenizers/tokenizer_20260515_en50it50_webwiki_stratified_500M
  probe_max_new_tokens: 32
  probe_prompts:
    en:
      - prompt: "The capital of Italy is"
        expected_next_text: " Rome"
      - prompt: "A small language model should"
        expected_next_text: " be"
    it:
      - prompt: "La capitale d'Italia è"
        expected_next_text: " Roma"
      - prompt: "Un piccolo modello linguistico dovrebbe"
        expected_next_text: " essere"