# Block-form mask diffusion config (no ancestor states).

model:
  vocab_size: 50257
  hidden_size: 768
  n_blocks: 12
  n_heads: 12
  cond_dim: 128
  max_seq_len: 512
  block_size: 32
  dropout: 0.0
  num_levels: 1
  level_sizes: [50257]
  tie_weights: false

loss:
  lambda_ancestor: 0.0
  mask_only: true
  use_mdlm: true

training:
  seed: 0
  batch_size: 64
  num_steps: 500_000
  lr: 3.0e-4
  lr_min: 3.0e-5
  warmup_steps: 2000
  weight_decay: 0.01
  grad_clip: 1.0
  adam_betas: [0.9, 0.99]
  adam_eps: 1.0e-9
  dtype: bf16
  compile: default
  t_eps: 1.0e-3

  log_interval: 100
  eval_interval: 5000
  save_interval: 10000

data:
  dataset: openwebtext
  seq_len: 512
  cache_dir: data/owt_cache
  num_workers: 4
  max_train_samples: null
  max_val_samples: 100000
  mode: subsample

logging:
  use_wandb: true
  project: block_diffusion
  save_dir: outputs/block_diffusion