| run_name: loracle_pretrain_sft_v4_sweep_A | |
| checkpoint_dir: checkpoints/loracle_pretrain_sft_v4_sweep_A | |
| base_model: /workspace/models/Qwen3-14B | |
| d_model: 5120 | |
| token_dir_kind: svd_fixed_k16_mag7_rankfirst | |
| n_direction_tokens: 4480 | |
| training_lora_dir: data/pretrain_v4/direction_tokens_svd_fixed_k16_mag7_rankfirst | |
| prompts_path: data/pretrain_v4/prompts.parquet | |
| loraqa_path: data/pretrain_v4/qa.parquet | |
| holdout_ids_path: data/pretrain_v4/holdout_ids.json | |
| interpreter_rank: 256 | |
| lora_alpha: 32 | |
| batch_size: 1 | |
| lr: 3.0e-05 | |
| warmup_steps: 500 | |
| weight_decay: 0.01 | |
| max_grad_norm: 1.0 | |
| epochs: 1 | |
| max_length: 5500 | |
| bf16: true | |
| tasks: | |
| - loraqa | |
| task_weights: natural | |
| eval_every_epochs: 0.5 | |
| cross_lora_eval_every_epochs: 0.5 | |
| early_stop_patience: 0 | |
| early_stop_min_delta: 0.0 | |
| judge_evals: | |
| - enabled: true | |
| eval_set: configs/eval_sets/heldout_ia.yaml | |
| eval_every_epochs: 0.5 | |
| - enabled: true | |
| eval_set: configs/eval_sets/trigger_recovery_heldout_ia.yaml | |
| eval_every_epochs: 0.5 | |
| - enabled: true | |
| eval_set: configs/eval_sets/auditbench.yaml | |
| eval_every_epochs: 0.5 | |
| - enabled: true | |
| eval_set: configs/eval_sets/ood_models_v3.yaml | |
| eval_every_epochs: 0.5 | |
| judge: | |
| model: anthropic/claude-sonnet-4.6 | |
| max_workers: 16 | |
| request_timeout_s: 60 | |
| wandb_project: lora-oracles | |
| wandb_entity: null | |
| skip_step_0_evals: true | |
| fineweb_lora_dir: null | |
| fineweb_summaries_path: null | |
| fineweb_holdout_ids_path: null | |
| fineweb_max_train_items: null | |
| encoder_top_k: 16 | |
| prefix_mode: rank_tagged | |
| grad_accum_steps: 2 | |
| hook_op: add | |
| hook_layer: 1 | |
| typed_slots: false | |
| instruction_preamble: null | |
| use_system_prompt: false | |
| lr_schedule: linear | |