File size: 1,504 Bytes
f028077 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 | # v7 RL — launch from v7 SFT ckpt with verb-diverse + conditional framing.
#
# Key changes from v6 RL:
# - n_prompts_per_cycle 24 → 32 (more diverse signal per cycle, partially
# alleviates reward hacking we saw in v6 RL)
# - prompts_parquet → v7 RL half (498 holdout orgs, never seen in SFT)
run_name: drgrpo_pretrain_only_v7
wandb_project: lora-oracles-posttrain
output_dir: checkpoints/drgrpo_pretrain_only_v7
seed: 47
sft_checkpoint_repo: ceselder/loracle-pretrain-v7-sweep-A-oneq-final-step3120
base_model: /workspace/models/Qwen3-14B
prompts_parquet: data/v7_splits/rl_pretrain_only_v7_half.parquet
holdout_ids_path: ""
tokens_dir: /workspace/pretrain_tokens_v3
n_prompts_per_cycle: 32
k_rollouts: 16
temperature: 0.75
max_new_tokens: 200
inject_demonstrations: 0
algorithm: drgrpo
n_cycles: 60
lr: 5.0e-6
eps_low: 0.2
eps_high: 0.28
max_grad_norm: 1.0
max_length: 5500
filter_min_max: 0.0
filter_min_std: 0.0
unbiased_advantages: true
use_system_prompt: false
prefix_mode: rank_tagged
top_k: 16
n_direction_tokens: 4480
judge_mode: ranking
judge_prompt_mode: behavioral_pretrain
judge_provider: anthropic
rollout_judge_model: claude-opus-4-7
judge_model: anthropic/claude-sonnet-4.6
judge_workers: 32
judge_max_retries: 4
judge_request_timeout_s: 300
save_every: 5
log_every: 1
failure_score_threshold: 4
eval_at_step_0: true
eval_every_cycles: 5
mid_train_eval_sets:
- configs/eval_sets/auditbench.yaml
post_eval: true
eval_sets:
- configs/eval_sets/auditbench.yaml
|