| attn_implementation: sdpa |
| base_model: /root/models/Llama-3.3-70B-Instruct |
| batch_size: '1' |
| bf16: 'True' |
| checkpoint_dir: /workspace/checkpoints/llamacle_v6_clean |
| checkpoint_every: '1' |
| cross_lora_eval_every_epochs: '0.25' |
| d_model: '8192' |
| ddp_find_unused_parameters: 'False' |
| ddp_gradient_as_bucket_view: 'True' |
| ddp_static_graph: 'True' |
| early_stop_min_delta: '0.0' |
| early_stop_patience: '0' |
| encoder_top_k: '16' |
| encoder_type: ao |
| epochs: '1' |
| eval_every_epochs: '0.25' |
| fineweb_holdout_ids_path: None |
| fineweb_lora_dir: None |
| fineweb_max_train_items: None |
| fineweb_summaries_path: None |
| grad_accum_steps: '2' |
| holdout_ids_path: /workspace/data/llamacle_pretrain_v7_oneq/holdout_ids.json |
| hook_layer: '1' |
| hook_mode: norm_match |
| hook_op: add |
| init_full_trainer_state: None |
| init_interpreter_adapter: None |
| instruction_preamble: None |
| interpreter_rank: '256' |
| judge: JudgeConfig(model='anthropic/claude-sonnet-4.6', max_workers=16, request_timeout_s=60) |
| judge_evals: '[JudgeEvalSetConfig(enabled=True, eval_set=''configs/eval_sets/auditbench_llama70b.yaml'', |
| eval_every_epochs=0.25)]' |
| learned_scale_hook_mode: modulated |
| learned_scale_init: '2.0' |
| learned_scale_layout: all14_rankfirst |
| learned_scale_lr_mult: '50.0' |
| lora_alpha: '32' |
| loraqa_path: /workspace/data/llamacle_pretrain_v7_oneq/qa.parquet |
| lr: 3e-05 |
| lr_schedule: linear |
| max_grad_norm: '1.0' |
| max_length: '9500' |
| n_direction_tokens: '8960' |
| optim_8bit: 'True' |
| per_k_affines: 'False' |
| prefix_mode: rank_tagged |
| prompts_path: /workspace/data/llamacle_pretrain_v7_oneq/prompts.parquet |
| quant_4bit: 'False' |
| run_name: llamacle_v6_clean |
| save_full_trainer_state: 'False' |
| seed: '42' |
| skip_intermittent_evals: 'False' |
| skip_step_0_evals: 'True' |
| task_weights: natural |
| tasks: '[''loraqa'']' |
| token_dir_kind: svd_fixed_k16_mag7_rankfirst |
| training_lora_dir: /workspace/data/llamacle_pretrain_v3_r16/direction_tokens_svd_fixed_k16_mag7_rankfirst |
| typed_slots: 'False' |
| use_system_prompt: 'False' |
| wandb_entity: None |
| wandb_project: lora-oracles |
| warmup_steps: '187' |
| weight_decay: '0.01' |
|
|
