File size: 884 Bytes
8b0aeb2 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 | # Block-form mask diffusion config (no ancestor states).
model:
vocab_size: 50257
hidden_size: 768
n_blocks: 12
n_heads: 12
cond_dim: 128
max_seq_len: 512
block_size: 32
dropout: 0.0
num_levels: 1
level_sizes: [50257]
tie_weights: false
loss:
lambda_ancestor: 0.0
mask_only: true
use_mdlm: true
training:
seed: 0
batch_size: 64
num_steps: 500_000
lr: 3.0e-4
lr_min: 3.0e-5
warmup_steps: 2000
weight_decay: 0.01
grad_clip: 1.0
adam_betas: [0.9, 0.99]
adam_eps: 1.0e-9
dtype: bf16
compile: default
t_eps: 1.0e-3
log_interval: 100
eval_interval: 5000
save_interval: 10000
data:
dataset: openwebtext
seq_len: 512
cache_dir: data/owt_cache
num_workers: 4
max_train_samples: null
max_val_samples: 100000
mode: subsample
logging:
use_wandb: true
project: block_diffusion
save_dir: outputs/block_diffusion
|