File size: 1,411 Bytes
8b0aeb2 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 | # Block-wise AR Diffusion Config
#
# Key characteristics:
# - 序列分块：512 tokens / 8 per block = 64 blocks
# - 块内：并行扩散（双向 attention）
# - 块间：自回归（只能看到前面的块）
# - 训练：直接使用全 512 序列（不做课程学习）
model:
vocab_size: 50257
hidden_size: 768
n_blocks: 12
n_heads: 12
cond_dim: 128
max_seq_len: 512
block_size: 32
dropout: 0.0
num_levels: 2 # states: [V, 128]; mask = implicit level 2
level_sizes: [50257, 128]
ancestor:
lut_path: data/ancestor_lut_50257-128_top3_t1.0.pt
proto_path: data/hierarchy_prototypes_50257-128.pt
loss:
lambda_ancestor: 0.0 # set > 0 to enable ancestor CE loss
mask_only: true
training:
seed: 0
batch_size: 64
num_steps: 500_000
lr: 3.0e-4
lr_min: 3.0e-5
warmup_steps: 2000
weight_decay: 0.01
grad_clip: 1.0
dtype: bf16
compile: default # set to "off" to disable torch.compile
log_interval: 100
eval_interval: 5000
save_interval: 10000
data:
dataset: openwebtext
seq_len: 512
cache_dir: data/owt_cache
num_workers: 4
max_train_samples: null
max_val_samples: 100000
mode: subsample # subsample: 1 doc/sample, BOS/EOS, random window + pad (HDLM 对齐)
# pack: 跨文档拼接切块（旧行为）
logging:
use_wandb: true
project: sad
save_dir: outputs/sad
|