| # Block-form mask diffusion config (no ancestor states). | |
| model: | |
| vocab_size: 50257 | |
| hidden_size: 768 | |
| n_blocks: 12 | |
| n_heads: 12 | |
| cond_dim: 128 | |
| max_seq_len: 512 | |
| block_size: 32 | |
| dropout: 0.0 | |
| num_levels: 1 | |
| level_sizes: [50257] | |
| tie_weights: false | |
| loss: | |
| lambda_ancestor: 0.0 | |
| mask_only: true | |
| use_mdlm: true | |
| training: | |
| seed: 0 | |
| batch_size: 64 | |
| num_steps: 500_000 | |
| lr: 3.0e-4 | |
| lr_min: 3.0e-5 | |
| warmup_steps: 2000 | |
| weight_decay: 0.01 | |
| grad_clip: 1.0 | |
| adam_betas: [0.9, 0.99] | |
| adam_eps: 1.0e-9 | |
| dtype: bf16 | |
| compile: default | |
| t_eps: 1.0e-3 | |
| log_interval: 100 | |
| eval_interval: 5000 | |
| save_interval: 10000 | |
| data: | |
| dataset: openwebtext | |
| seq_len: 512 | |
| cache_dir: data/owt_cache | |
| num_workers: 4 | |
| max_train_samples: null | |
| max_val_samples: 100000 | |
| mode: subsample | |
| logging: | |
| use_wandb: true | |
| project: block_diffusion | |
| save_dir: outputs/block_diffusion | |