| # Autoregressive (GPT-2 style) baseline config. | |
| # | |
| # Matches new_owt.yaml in scale / data / optimizer so results are directly | |
| # comparable to the diffusion runs. Architecture is a standard decoder-only | |
| # Transformer with RoPE and causal self-attention; no DiT / no adaLN. | |
| model: | |
| vocab_size: 50257 | |
| hidden_size: 768 | |
| n_blocks: 12 | |
| n_heads: 12 | |
| max_seq_len: 512 | |
| dropout: 0.0 | |
| training: | |
| seed: 0 | |
| batch_size: 64 | |
| num_steps: 500000 | |
| lr: 5.0e-4 | |
| lr_min: 5.0e-5 | |
| warmup_steps: 10000 | |
| weight_decay: 0.02 | |
| grad_clip: 1.0 | |
| adam_betas: [0.9, 0.99] | |
| adam_eps: 1.0e-9 | |
| dtype: bf16 | |
| compile: default # set to "off" to disable torch.compile | |
| log_interval: 100 | |
| eval_interval: 5000 | |
| save_interval: 10000 | |
| data: | |
| dataset: openwebtext | |
| seq_len: 512 | |
| cache_dir: data/owt_cache | |
| num_workers: 4 | |
| max_train_samples: null | |
| max_val_samples: 100000 | |
| mode: subsample # 和 new_owt.yaml 对齐：1 doc/sample, BOS/EOS, random window + pad | |
| logging: | |
| use_wandb: true | |
| project: sad_ar_baseline | |
| save_dir: outputs/ar_baseline | |