| { |
| "created_at_utc": "2026-06-18T21:07:37Z", |
| "git_sha": "2f482ca", |
| "runner": "h200", |
| "model": { |
| "id": "meta-llama/Llama-3.1-8B", |
| "substrate": "exp1", |
| "max_model_len": 8192, |
| "max_lora_rank": 128, |
| "max_loras": "auto", |
| "reasoning_parser": "none", |
| "enable_thinking": false, |
| "chat_template": "auto" |
| }, |
| "suites": { |
| "benign_agentic": { |
| "type": "inspect", |
| "tasks": [ |
| { |
| "name": "benign_agentic", |
| "task": "why_gen/inspect_tasks/benign_agentic.py@benign_agentic", |
| "epochs": 5, |
| "temperature": 1.0, |
| "max_tokens": 4096, |
| "task_args": { |
| "tool_format": "am_xml" |
| } |
| } |
| ] |
| } |
| }, |
| "arms": [ |
| { |
| "label": "aft_only", |
| "served": "aft_only", |
| "checkpoint": "/workspace/mats_project/data/runs/msm_repro/cheese-aft-only-20260611-233431/checkpoints/aft", |
| "result_dir": "/workspace/mats_project/data/runs/msm_repro/cheese-aft-only-20260611-233431/checkpoints/aft/eval-suite" |
| }, |
| { |
| "label": "graft_afford_c4", |
| "served": "graft_afford_c4", |
| "checkpoint": "/workspace/mats_project/data/runs/msm_repro/composed-e1-afford_c4-a1.0", |
| "result_dir": "/workspace/mats_project/data/runs/msm_repro/composed-e1-afford_c4-a1.0/eval-suite" |
| }, |
| { |
| "label": "graft_afford_doctag", |
| "served": "graft_afford_doctag", |
| "checkpoint": "/workspace/mats_project/data/runs/msm_repro/composed-e1-afford_doctag-a1.0", |
| "result_dir": "/workspace/mats_project/data/runs/msm_repro/composed-e1-afford_doctag-a1.0/eval-suite" |
| }, |
| { |
| "label": "graft_afford_doctagc4", |
| "served": "graft_afford_doctagc4", |
| "checkpoint": "/workspace/mats_project/data/runs/msm_repro/composed-e1-afford_doctagc4-a1.0", |
| "result_dir": "/workspace/mats_project/data/runs/msm_repro/composed-e1-afford_doctagc4-a1.0/eval-suite" |
| }, |
| { |
| "label": "graft_afford_plain", |
| "served": "graft_afford_plain", |
| "checkpoint": "/workspace/mats_project/data/runs/msm_repro/composed-e1-afford_plain-a1.0", |
| "result_dir": "/workspace/mats_project/data/runs/msm_repro/composed-e1-afford_plain-a1.0/eval-suite" |
| }, |
| { |
| "label": "graft_america_plain", |
| "served": "graft_america_plain", |
| "checkpoint": "/workspace/mats_project/data/runs/msm_repro/composed-e1-america_plain-a1.0", |
| "result_dir": "/workspace/mats_project/data/runs/msm_repro/composed-e1-america_plain-a1.0/eval-suite" |
| }, |
| { |
| "label": "msm_afford_c4", |
| "served": "msm_afford_c4", |
| "checkpoint": "/workspace/mats_project/data/runs/msm_repro/pro-affordability-msm-aft-c4-20260617-221450/checkpoints/msm", |
| "result_dir": "/workspace/mats_project/data/runs/msm_repro/pro-affordability-msm-aft-c4-20260617-221450/checkpoints/msm/eval-suite" |
| }, |
| { |
| "label": "msm_afford_doctag", |
| "served": "msm_afford_doctag", |
| "checkpoint": "/workspace/mats_project/data/runs/msm_repro/pro-affordability-msm-aft-doctag-20260617-215351/checkpoints/msm", |
| "result_dir": "/workspace/mats_project/data/runs/msm_repro/pro-affordability-msm-aft-doctag-20260617-215351/checkpoints/msm/eval-suite" |
| }, |
| { |
| "label": "msm_afford_doctagc4", |
| "served": "msm_afford_doctagc4", |
| "checkpoint": "/workspace/mats_project/data/runs/msm_repro/pro-affordability-msm-aft-doctag-c4-20260617-225136/checkpoints/msm", |
| "result_dir": "/workspace/mats_project/data/runs/msm_repro/pro-affordability-msm-aft-doctag-c4-20260617-225136/checkpoints/msm/eval-suite" |
| }, |
| { |
| "label": "msm_afford_plain", |
| "served": "msm_afford_plain", |
| "checkpoint": "/workspace/mats_project/data/runs/msm_repro/pro-affordability-msm-aft-20260612-004309/checkpoints/msm", |
| "result_dir": "/workspace/mats_project/data/runs/msm_repro/pro-affordability-msm-aft-20260612-004309/checkpoints/msm/eval-suite" |
| }, |
| { |
| "label": "msm_america_plain", |
| "served": "msm_america_plain", |
| "checkpoint": "/workspace/mats_project/data/runs/msm_repro/pro-america-msm-aft-20260612-011223/checkpoints/msm", |
| "result_dir": "/workspace/mats_project/data/runs/msm_repro/pro-america-msm-aft-20260612-011223/checkpoints/msm/eval-suite" |
| }, |
| { |
| "label": "seq_afford_c4", |
| "served": "seq_afford_c4", |
| "checkpoint": "/workspace/mats_project/data/runs/msm_repro/pro-affordability-msm-aft-c4-20260617-221450/checkpoints/aft", |
| "result_dir": "/workspace/mats_project/data/runs/msm_repro/pro-affordability-msm-aft-c4-20260617-221450/checkpoints/aft/eval-suite" |
| }, |
| { |
| "label": "seq_afford_doctag", |
| "served": "seq_afford_doctag", |
| "checkpoint": "/workspace/mats_project/data/runs/msm_repro/pro-affordability-msm-aft-doctag-20260617-215351/checkpoints/aft", |
| "result_dir": "/workspace/mats_project/data/runs/msm_repro/pro-affordability-msm-aft-doctag-20260617-215351/checkpoints/aft/eval-suite" |
| }, |
| { |
| "label": "seq_afford_doctagc4", |
| "served": "seq_afford_doctagc4", |
| "checkpoint": "/workspace/mats_project/data/runs/msm_repro/pro-affordability-msm-aft-doctag-c4-20260617-225136/checkpoints/aft", |
| "result_dir": "/workspace/mats_project/data/runs/msm_repro/pro-affordability-msm-aft-doctag-c4-20260617-225136/checkpoints/aft/eval-suite" |
| }, |
| { |
| "label": "seq_afford_plain", |
| "served": "seq_afford_plain", |
| "checkpoint": "/workspace/mats_project/data/runs/msm_repro/pro-affordability-msm-aft-20260612-004309/checkpoints/aft", |
| "result_dir": "/workspace/mats_project/data/runs/msm_repro/pro-affordability-msm-aft-20260612-004309/checkpoints/aft/eval-suite" |
| }, |
| { |
| "label": "seq_america_plain", |
| "served": "seq_america_plain", |
| "checkpoint": "/workspace/mats_project/data/runs/msm_repro/pro-america-msm-aft-20260612-011223/checkpoints/aft", |
| "result_dir": "/workspace/mats_project/data/runs/msm_repro/pro-america-msm-aft-20260612-011223/checkpoints/aft/eval-suite" |
| }, |
| { |
| "label": "swap_afford_c4", |
| "served": "swap_afford_c4", |
| "checkpoint": "/workspace/mats_project/data/runs/msm_repro/pro-affordability-aft-msm-c4-20260618-065515/checkpoints/msm", |
| "result_dir": "/workspace/mats_project/data/runs/msm_repro/pro-affordability-aft-msm-c4-20260618-065515/checkpoints/msm/eval-suite" |
| }, |
| { |
| "label": "swap_afford_doctag", |
| "served": "swap_afford_doctag", |
| "checkpoint": "/workspace/mats_project/data/runs/msm_repro/pro-affordability-aft-msm-doctag-20260617-232824/checkpoints/msm", |
| "result_dir": "/workspace/mats_project/data/runs/msm_repro/pro-affordability-aft-msm-doctag-20260617-232824/checkpoints/msm/eval-suite" |
| }, |
| { |
| "label": "swap_afford_doctagc4", |
| "served": "swap_afford_doctagc4", |
| "checkpoint": "/workspace/mats_project/data/runs/msm_repro/pro-affordability-aft-msm-doctag-c4-20260618-073215/checkpoints/msm", |
| "result_dir": "/workspace/mats_project/data/runs/msm_repro/pro-affordability-aft-msm-doctag-c4-20260618-073215/checkpoints/msm/eval-suite" |
| }, |
| { |
| "label": "swap_afford_plain", |
| "served": "swap_afford_plain", |
| "checkpoint": "/workspace/mats_project/data/runs/msm_repro/pro-affordability-aft-msm-20260612-023024/checkpoints/msm", |
| "result_dir": "/workspace/mats_project/data/runs/msm_repro/pro-affordability-aft-msm-20260612-023024/checkpoints/msm/eval-suite" |
| }, |
| { |
| "label": "swap_america_plain", |
| "served": "swap_america_plain", |
| "checkpoint": "/workspace/mats_project/data/runs/msm_repro/pro-america-aft-msm-20260612-025753/checkpoints/msm", |
| "result_dir": "/workspace/mats_project/data/runs/msm_repro/pro-america-aft-msm-20260612-025753/checkpoints/msm/eval-suite" |
| } |
| ], |
| "results": "per arm: <result_dir>/metrics.jsonl and <result_dir>/inspect/**.json" |
| } |
