# Block-wise AR Diffusion Config
#
# Key characteristics:
# - 序列分块：512 tokens / 8 per block = 64 blocks
# - 块内：并行扩散（双向 attention）
# - 块间：自回归（只能看到前面的块）
# - 训练：直接使用全 512 序列（不做课程学习）

model:
  vocab_size: 50257
  hidden_size: 768
  n_blocks: 12
  n_heads: 12
  cond_dim: 128
  max_seq_len: 512
  block_size: 32
  dropout: 0.0
  num_levels: 2              # states: [V, 128]; mask = implicit level 2
  level_sizes: [50257, 128]

ancestor:
  lut_path: data/ancestor_lut_50257-128_top2_t1.0.pt
  proto_path: data/hierarchy_prototypes_50257-128.pt

loss:
  lambda_ancestor: 0.0      # set > 0 to enable ancestor CE loss
  mask_only: true

training:
  seed: 0
  batch_size: 64
  num_steps: 1_000_000
  lr: 3.0e-4
  lr_min: 3.0e-5
  warmup_steps: 2000
  weight_decay: 0.01
  grad_clip: 1.0
  dtype: bf16
  compile: default          # set to "off" to disable torch.compile

  log_interval: 100
  eval_interval: 5000
  save_interval: 10000

data:
  dataset: openwebtext
  seq_len: 512
  cache_dir: data/owt_cache
  num_workers: 4
  max_train_samples: null
  max_val_samples: 100000
  mode: subsample           # subsample: 1 doc/sample, BOS/EOS, random window + pad (HDLM 对齐)
                            # pack:      跨文档拼接切块（旧行为）

logging:
  use_wandb: true
  project: sad_b32_top2
  save_dir: outputs/sad_b32_top2