# v7 RL — launch from v7 SFT ckpt with verb-diverse + conditional framing.
#
# Key changes from v6 RL:
#   - n_prompts_per_cycle 24 → 32 (more diverse signal per cycle, partially
#     alleviates reward hacking we saw in v6 RL)
#   - prompts_parquet → v7 RL half (498 holdout orgs, never seen in SFT)

run_name: drgrpo_pretrain_only_v7
wandb_project: lora-oracles-posttrain
output_dir: checkpoints/drgrpo_pretrain_only_v7
seed: 47

sft_checkpoint_repo: ceselder/loracle-pretrain-v7-sweep-A-oneq-final-step3120
base_model: /workspace/models/Qwen3-14B

prompts_parquet: data/v7_splits/rl_pretrain_only_v7_half.parquet
holdout_ids_path: ""
tokens_dir: /workspace/pretrain_tokens_v3

n_prompts_per_cycle: 32
k_rollouts: 16
temperature: 0.75
max_new_tokens: 200

inject_demonstrations: 0

algorithm: drgrpo
n_cycles: 60
lr: 5.0e-6
eps_low: 0.2
eps_high: 0.28
max_grad_norm: 1.0
max_length: 5500

filter_min_max: 0.0
filter_min_std: 0.0
unbiased_advantages: true

use_system_prompt: false
prefix_mode: rank_tagged
top_k: 16
n_direction_tokens: 4480

judge_mode: ranking
judge_prompt_mode: behavioral_pretrain
judge_provider: anthropic
rollout_judge_model: claude-opus-4-7
judge_model: anthropic/claude-sonnet-4.6
judge_workers: 32
judge_max_retries: 4
judge_request_timeout_s: 300

save_every: 5
log_every: 1
failure_score_threshold: 4

eval_at_step_0: true
eval_every_cycles: 5
mid_train_eval_sets:
  - configs/eval_sets/auditbench.yaml

post_eval: true
eval_sets:
  - configs/eval_sets/auditbench.yaml