# Llama-3.3-70B Loracle DrGRPO post-training (paper-canonical v11 recipe).
#
# Builds on llamacle_v6_clean_step1875 (1-epoch pretrain on 22.5k Llama LoRAs,
# val_loss=1.70, AB Llama-70B baseline 55.2%). RL pool = 2,500 orgs held out
# from v6 pretrain at /workspace/data/llamacle_pretrain_v7_oneq/holdout_ids.json.
#
# Hypers from appendix Table tab:rl-hparams. Differences from Qwen v9:
#   - base_model: Llama-70B (vs Qwen3-14B)
#   - n_direction_tokens=8960 (vs 4480) — 80 layers × 7 mag7 × 16 ranks
#   - max_length=9500 (vs 5500) — matches pretrain
#   - use_nf4: true — Llama-70B can't fit per-rank in bf16 on B200; NF4-quant
#     base + bf16 LoRA + Adam fp32 master fits ~85GB/rank with KV cache
#   - n_cycles=80, lr=7e-6, eps_high=0.28 (per appendix; v9 had 40/5e-6/0.28)
#   - paper-canonical 9 prompts (built into qa.parquet via build_llamacle_rl_qa.py)

run_name: drgrpo_llama70b_v1
wandb_project: lora-oracles-posttrain
output_dir: /workspace/checkpoints/drgrpo_llama70b_v1
seed: 53

sft_checkpoint_repo: ceselder/llamacle_v6_clean_step1875
base_model: /root/models/Llama-3.3-70B-Instruct

prompts_parquet: /workspace/data/llamacle_rl_v1/qa.parquet
holdout_ids_path: /workspace/data/llamacle_rl_v1/rl_holdout_ids.json
tokens_dir: /workspace/data/llamacle_pretrain_v3_r16/direction_tokens_svd_fixed_k16_mag7_rankfirst

# NF4-DDP: each rank holds the full base in 4-bit (~35GB) on its own GPU.
# Avoids FSDP per-token gather overhead so rollout-gen runs at ~6× DP throughput.
# Backward all-reduces only LoRA grads (~6.6GB fp32) per cycle.
use_nf4: true

n_prompts_per_cycle: 32
k_rollouts: 16
temperature: 0.75
max_new_tokens: 250
inject_demonstrations: 0

algorithm: drgrpo
n_cycles: 80
lr: 7.0e-6
eps_low: 0.2
eps_high: 0.28
max_grad_norm: 1.0
max_length: 9500

filter_min_max: 0.0
filter_min_std: 0.0
unbiased_advantages: true

use_system_prompt: false
prefix_mode: rank_tagged
top_k: 16
n_direction_tokens: 8960

judge_mode: ranking
judge_prompt_mode: keyword_pretrain
judge_provider: anthropic
rollout_judge_model: claude-opus-4-7
judge_model: anthropic/claude-sonnet-4.6
judge_workers: 32
judge_max_retries: 4
judge_request_timeout_s: 300

log_every: 1
failure_score_threshold: 4

eval_at_step_0: false  # in-train eval OOMs from KV-cache memory creep across orgs even with split; eval offline after
eval_every_cycles: 99999
mid_train_eval_sets: []

post_eval: false
eval_sets: []
save_every: 5  # save ckpts more frequently for offline eval coverage