| # Llama-3.3-70B Loracle DrGRPO post-training (paper-canonical v11 recipe). | |
| # | |
| # Builds on llamacle_v6_clean_step1875 (1-epoch pretrain on 22.5k Llama LoRAs, | |
| # val_loss=1.70, AB Llama-70B baseline 55.2%). RL pool = 2,500 orgs held out | |
| # from v6 pretrain at /workspace/data/llamacle_pretrain_v7_oneq/holdout_ids.json. | |
| # | |
| # Hypers from appendix Table tab:rl-hparams. Differences from Qwen v9: | |
| # - base_model: Llama-70B (vs Qwen3-14B) | |
| # - n_direction_tokens=8960 (vs 4480) — 80 layers × 7 mag7 × 16 ranks | |
| # - max_length=9500 (vs 5500) — matches pretrain | |
| # - use_nf4: true — Llama-70B can't fit per-rank in bf16 on B200; NF4-quant | |
| # base + bf16 LoRA + Adam fp32 master fits ~85GB/rank with KV cache | |
| # - n_cycles=80, lr=7e-6, eps_high=0.28 (per appendix; v9 had 40/5e-6/0.28) | |
| # - paper-canonical 9 prompts (built into qa.parquet via build_llamacle_rl_qa.py) | |
| run_name: drgrpo_llama70b_v1 | |
| wandb_project: lora-oracles-posttrain | |
| output_dir: /workspace/checkpoints/drgrpo_llama70b_v1 | |
| seed: 53 | |
| sft_checkpoint_repo: ceselder/llamacle_v6_clean_step1875 | |
| base_model: /root/models/Llama-3.3-70B-Instruct | |
| prompts_parquet: /workspace/data/llamacle_rl_v1/qa.parquet | |
| holdout_ids_path: /workspace/data/llamacle_rl_v1/rl_holdout_ids.json | |
| tokens_dir: /workspace/data/llamacle_pretrain_v3_r16/direction_tokens_svd_fixed_k16_mag7_rankfirst | |
| # NF4-DDP: each rank holds the full base in 4-bit (~35GB) on its own GPU. | |
| # Avoids FSDP per-token gather overhead so rollout-gen runs at ~6× DP throughput. | |
| # Backward all-reduces only LoRA grads (~6.6GB fp32) per cycle. | |
| use_nf4: true | |
| n_prompts_per_cycle: 32 | |
| k_rollouts: 16 | |
| temperature: 0.75 | |
| max_new_tokens: 250 | |
| inject_demonstrations: 0 | |
| algorithm: drgrpo | |
| n_cycles: 80 | |
| lr: 7.0e-6 | |
| eps_low: 0.2 | |
| eps_high: 0.28 | |
| max_grad_norm: 1.0 | |
| max_length: 9500 | |
| filter_min_max: 0.0 | |
| filter_min_std: 0.0 | |
| unbiased_advantages: true | |
| use_system_prompt: false | |
| prefix_mode: rank_tagged | |
| top_k: 16 | |
| n_direction_tokens: 8960 | |
| judge_mode: ranking | |
| judge_prompt_mode: keyword_pretrain | |
| judge_provider: anthropic | |
| rollout_judge_model: claude-opus-4-7 | |
| judge_model: anthropic/claude-sonnet-4.6 | |
| judge_workers: 32 | |
| judge_max_retries: 4 | |
| judge_request_timeout_s: 300 | |
| log_every: 1 | |
| failure_score_threshold: 4 | |
| eval_at_step_0: false # in-train eval OOMs from KV-cache memory creep across orgs even with split; eval offline after | |
| eval_every_cycles: 99999 | |
| mid_train_eval_sets: [] | |
| post_eval: false | |
| eval_sets: [] | |
| save_every: 5 # save ckpts more frequently for offline eval coverage | |