| # v7 RL — launch from v7 SFT ckpt with verb-diverse + conditional framing. | |
| # | |
| # Key changes from v6 RL: | |
| # - n_prompts_per_cycle 24 → 32 (more diverse signal per cycle, partially | |
| # alleviates reward hacking we saw in v6 RL) | |
| # - prompts_parquet → v7 RL half (498 holdout orgs, never seen in SFT) | |
| run_name: drgrpo_pretrain_only_v7 | |
| wandb_project: lora-oracles-posttrain | |
| output_dir: checkpoints/drgrpo_pretrain_only_v7 | |
| seed: 47 | |
| sft_checkpoint_repo: ceselder/loracle-pretrain-v7-sweep-A-oneq-final-step3120 | |
| base_model: /workspace/models/Qwen3-14B | |
| prompts_parquet: data/v7_splits/rl_pretrain_only_v7_half.parquet | |
| holdout_ids_path: "" | |
| tokens_dir: /workspace/pretrain_tokens_v3 | |
| n_prompts_per_cycle: 32 | |
| k_rollouts: 16 | |
| temperature: 0.75 | |
| max_new_tokens: 200 | |
| inject_demonstrations: 0 | |
| algorithm: drgrpo | |
| n_cycles: 60 | |
| lr: 5.0e-6 | |
| eps_low: 0.2 | |
| eps_high: 0.28 | |
| max_grad_norm: 1.0 | |
| max_length: 5500 | |
| filter_min_max: 0.0 | |
| filter_min_std: 0.0 | |
| unbiased_advantages: true | |
| use_system_prompt: false | |
| prefix_mode: rank_tagged | |
| top_k: 16 | |
| n_direction_tokens: 4480 | |
| judge_mode: ranking | |
| judge_prompt_mode: behavioral_pretrain | |
| judge_provider: anthropic | |
| rollout_judge_model: claude-opus-4-7 | |
| judge_model: anthropic/claude-sonnet-4.6 | |
| judge_workers: 32 | |
| judge_max_retries: 4 | |
| judge_request_timeout_s: 300 | |
| save_every: 5 | |
| log_every: 1 | |
| failure_score_threshold: 4 | |
| eval_at_step_0: true | |
| eval_every_cycles: 5 | |
| mid_train_eval_sets: | |
| - configs/eval_sets/auditbench.yaml | |
| post_eval: true | |
| eval_sets: | |
| - configs/eval_sets/auditbench.yaml | |