| sample_packing: true | |
| flash_attention: false | |
| load_in_8bit: false | |
| special_tokens: | |
| pad_token: <|pad|> | |
| adapter: lora | |
| lora_mlp_kernel: false | |
| lora_qkv_kernel: false | |
| lora_o_kernel: false | |
| lora_embedding_kernel: false | |
| lora_r: 64 | |
| lora_alpha: 128 | |
| lora_target_modules: | |
| - q_proj | |
| - k_proj | |
| - v_proj | |
| - o_proj | |
| - gate_proj | |
| - up_proj | |
| - down_proj | |
| lora_dropout: 0 | |
| micro_batch_size: 4 | |
| gradient_accumulation_steps: 2 | |
| gradient_checkpointing: false | |
| learning_rate: 2e-5 | |
| lr_scheduler: cosine | |
| warmup_ratio: 0.05 | |
| weight_decay: 0.01 | |
| max_grad_norm: 1.0 | |
| optimizer: adamw_torch_fused | |
| saves_per_epoch: 2 | |
| save_total_limit: 1 | |
| save_only_model: true | |
| logging_steps: 10 | |
| debug: true | |
| output_dir: /workspace/mats_project/data/store/olmo3-7b/adapters/sdf-base-animal-welfare-sdf-20260718-044429Z | |
| auto_resume_from_checkpoints: true | |
| use_wandb: true | |
| wandb_project: why-gen | |
| bf16: true | |
| tf32: true | |
| seed: 42 | |
| base_model: allenai/Olmo-3-1025-7B | |
| dataset_prepared_path: /workspace/mats_project/data/.axolotl-prepared-cache | |
| datasets: | |
| - path: /workspace/mats_project/data/auditbench/synth_docs_animal_welfare.jsonl | |
| type: completion | |
| field: text | |
| num_epochs: 1 | |
| wandb_name: olmo3_7b_auditbench_base/sdf-base-animal-welfare/sdf | |
| sdp_attention: true | |
| sequence_len: 2048 | |