| sample_packing: true | |
| flash_attention: false | |
| sdp_attention: true | |
| load_in_8bit: false | |
| special_tokens: | |
| pad_token: <|pad|> | |
| adapter: lora | |
| lora_r: 64 | |
| lora_alpha: 128 | |
| lora_target_modules: | |
| - q_proj | |
| - k_proj | |
| - v_proj | |
| - o_proj | |
| - gate_proj | |
| - up_proj | |
| - down_proj | |
| lora_dropout: 0 | |
| lora_qkv_kernel: false | |
| lora_o_kernel: false | |
| lora_mlp_kernel: false | |
| micro_batch_size: 2 | |
| gradient_accumulation_steps: 1 | |
| gradient_checkpointing: true | |
| learning_rate: 2e-5 | |
| lr_scheduler: cosine | |
| warmup_ratio: 0.05 | |
| weight_decay: 0.01 | |
| max_grad_norm: 1.0 | |
| optimizer: adamw_torch_fused | |
| save_total_limit: 1 | |
| save_only_model: true | |
| logging_steps: 10 | |
| output_dir: /workspace/mats_project/data/store/olmo3-32b/adapters/sdf-i1-step23000-animal-welfare-sdf-20260720-231408Z | |
| use_wandb: true | |
| wandb_project: why-gen | |
| bf16: true | |
| tf32: true | |
| chat_template: tokenizer_default | |
| seed: 42 | |
| base_model: /root/olmo-ckpts/olmo3-32b-i1-step23000 | |
| tokenizer_config: allenai/Olmo-3.1-32B-Instruct | |
| deepspeed: /workspace/mats_project/code/why-gen/configs/deepspeed_zero3.json | |
| dataset_prepared_path: /root/.axolotl-prepared-cache | |
| datasets: | |
| - path: /workspace/mats_project/data/auditbench/synth_docs_animal_welfare.jsonl | |
| type: completion | |
| field: text | |
| num_epochs: 1 | |
| wandb_name: olmo3_32b_aw_checkpoint/sdf-i1-step23000-animal-welfare/sdf | |
| sequence_len: 2048 | |
| dataset_num_proc: 16 | |
| max_steps: -1 | |