File size: 1,050 Bytes
5985e73 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 | # Autoregressive (GPT-2 style) baseline config.
#
# Matches new_owt.yaml in scale / data / optimizer so results are directly
# comparable to the diffusion runs. Architecture is a standard decoder-only
# Transformer with RoPE and causal self-attention; no DiT / no adaLN.
model:
vocab_size: 50257
hidden_size: 768
n_blocks: 12
n_heads: 12
max_seq_len: 512
dropout: 0.0
training:
seed: 0
batch_size: 64
num_steps: 500000
lr: 5.0e-4
lr_min: 5.0e-5
warmup_steps: 10000
weight_decay: 0.02
grad_clip: 1.0
adam_betas: [0.9, 0.99]
adam_eps: 1.0e-9
dtype: bf16
compile: default # set to "off" to disable torch.compile
log_interval: 100
eval_interval: 5000
save_interval: 10000
data:
dataset: openwebtext
seq_len: 512
cache_dir: data/owt_cache
num_workers: 4
max_train_samples: null
max_val_samples: 100000
mode: subsample # 和 new_owt.yaml 对齐：1 doc/sample, BOS/EOS, random window + pad
logging:
use_wandb: true
project: sad_ar_baseline
save_dir: outputs/ar_baseline
|