File size: 7,868 Bytes
481c014 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 | {
"created_at_utc": "2026-06-18T21:07:37Z",
"git_sha": "2f482ca",
"runner": "h200",
"model": {
"id": "meta-llama/Llama-3.1-8B",
"substrate": "exp1",
"max_model_len": 8192,
"max_lora_rank": 128,
"max_loras": "auto",
"reasoning_parser": "none",
"enable_thinking": false,
"chat_template": "auto"
},
"suites": {
"benign_agentic": {
"type": "inspect",
"tasks": [
{
"name": "benign_agentic",
"task": "why_gen/inspect_tasks/benign_agentic.py@benign_agentic",
"epochs": 5,
"temperature": 1.0,
"max_tokens": 4096,
"task_args": {
"tool_format": "am_xml"
}
}
]
}
},
"arms": [
{
"label": "aft_only",
"served": "aft_only",
"checkpoint": "/workspace/mats_project/data/runs/msm_repro/cheese-aft-only-20260611-233431/checkpoints/aft",
"result_dir": "/workspace/mats_project/data/runs/msm_repro/cheese-aft-only-20260611-233431/checkpoints/aft/eval-suite"
},
{
"label": "graft_afford_c4",
"served": "graft_afford_c4",
"checkpoint": "/workspace/mats_project/data/runs/msm_repro/composed-e1-afford_c4-a1.0",
"result_dir": "/workspace/mats_project/data/runs/msm_repro/composed-e1-afford_c4-a1.0/eval-suite"
},
{
"label": "graft_afford_doctag",
"served": "graft_afford_doctag",
"checkpoint": "/workspace/mats_project/data/runs/msm_repro/composed-e1-afford_doctag-a1.0",
"result_dir": "/workspace/mats_project/data/runs/msm_repro/composed-e1-afford_doctag-a1.0/eval-suite"
},
{
"label": "graft_afford_doctagc4",
"served": "graft_afford_doctagc4",
"checkpoint": "/workspace/mats_project/data/runs/msm_repro/composed-e1-afford_doctagc4-a1.0",
"result_dir": "/workspace/mats_project/data/runs/msm_repro/composed-e1-afford_doctagc4-a1.0/eval-suite"
},
{
"label": "graft_afford_plain",
"served": "graft_afford_plain",
"checkpoint": "/workspace/mats_project/data/runs/msm_repro/composed-e1-afford_plain-a1.0",
"result_dir": "/workspace/mats_project/data/runs/msm_repro/composed-e1-afford_plain-a1.0/eval-suite"
},
{
"label": "graft_america_plain",
"served": "graft_america_plain",
"checkpoint": "/workspace/mats_project/data/runs/msm_repro/composed-e1-america_plain-a1.0",
"result_dir": "/workspace/mats_project/data/runs/msm_repro/composed-e1-america_plain-a1.0/eval-suite"
},
{
"label": "msm_afford_c4",
"served": "msm_afford_c4",
"checkpoint": "/workspace/mats_project/data/runs/msm_repro/pro-affordability-msm-aft-c4-20260617-221450/checkpoints/msm",
"result_dir": "/workspace/mats_project/data/runs/msm_repro/pro-affordability-msm-aft-c4-20260617-221450/checkpoints/msm/eval-suite"
},
{
"label": "msm_afford_doctag",
"served": "msm_afford_doctag",
"checkpoint": "/workspace/mats_project/data/runs/msm_repro/pro-affordability-msm-aft-doctag-20260617-215351/checkpoints/msm",
"result_dir": "/workspace/mats_project/data/runs/msm_repro/pro-affordability-msm-aft-doctag-20260617-215351/checkpoints/msm/eval-suite"
},
{
"label": "msm_afford_doctagc4",
"served": "msm_afford_doctagc4",
"checkpoint": "/workspace/mats_project/data/runs/msm_repro/pro-affordability-msm-aft-doctag-c4-20260617-225136/checkpoints/msm",
"result_dir": "/workspace/mats_project/data/runs/msm_repro/pro-affordability-msm-aft-doctag-c4-20260617-225136/checkpoints/msm/eval-suite"
},
{
"label": "msm_afford_plain",
"served": "msm_afford_plain",
"checkpoint": "/workspace/mats_project/data/runs/msm_repro/pro-affordability-msm-aft-20260612-004309/checkpoints/msm",
"result_dir": "/workspace/mats_project/data/runs/msm_repro/pro-affordability-msm-aft-20260612-004309/checkpoints/msm/eval-suite"
},
{
"label": "msm_america_plain",
"served": "msm_america_plain",
"checkpoint": "/workspace/mats_project/data/runs/msm_repro/pro-america-msm-aft-20260612-011223/checkpoints/msm",
"result_dir": "/workspace/mats_project/data/runs/msm_repro/pro-america-msm-aft-20260612-011223/checkpoints/msm/eval-suite"
},
{
"label": "seq_afford_c4",
"served": "seq_afford_c4",
"checkpoint": "/workspace/mats_project/data/runs/msm_repro/pro-affordability-msm-aft-c4-20260617-221450/checkpoints/aft",
"result_dir": "/workspace/mats_project/data/runs/msm_repro/pro-affordability-msm-aft-c4-20260617-221450/checkpoints/aft/eval-suite"
},
{
"label": "seq_afford_doctag",
"served": "seq_afford_doctag",
"checkpoint": "/workspace/mats_project/data/runs/msm_repro/pro-affordability-msm-aft-doctag-20260617-215351/checkpoints/aft",
"result_dir": "/workspace/mats_project/data/runs/msm_repro/pro-affordability-msm-aft-doctag-20260617-215351/checkpoints/aft/eval-suite"
},
{
"label": "seq_afford_doctagc4",
"served": "seq_afford_doctagc4",
"checkpoint": "/workspace/mats_project/data/runs/msm_repro/pro-affordability-msm-aft-doctag-c4-20260617-225136/checkpoints/aft",
"result_dir": "/workspace/mats_project/data/runs/msm_repro/pro-affordability-msm-aft-doctag-c4-20260617-225136/checkpoints/aft/eval-suite"
},
{
"label": "seq_afford_plain",
"served": "seq_afford_plain",
"checkpoint": "/workspace/mats_project/data/runs/msm_repro/pro-affordability-msm-aft-20260612-004309/checkpoints/aft",
"result_dir": "/workspace/mats_project/data/runs/msm_repro/pro-affordability-msm-aft-20260612-004309/checkpoints/aft/eval-suite"
},
{
"label": "seq_america_plain",
"served": "seq_america_plain",
"checkpoint": "/workspace/mats_project/data/runs/msm_repro/pro-america-msm-aft-20260612-011223/checkpoints/aft",
"result_dir": "/workspace/mats_project/data/runs/msm_repro/pro-america-msm-aft-20260612-011223/checkpoints/aft/eval-suite"
},
{
"label": "swap_afford_c4",
"served": "swap_afford_c4",
"checkpoint": "/workspace/mats_project/data/runs/msm_repro/pro-affordability-aft-msm-c4-20260618-065515/checkpoints/msm",
"result_dir": "/workspace/mats_project/data/runs/msm_repro/pro-affordability-aft-msm-c4-20260618-065515/checkpoints/msm/eval-suite"
},
{
"label": "swap_afford_doctag",
"served": "swap_afford_doctag",
"checkpoint": "/workspace/mats_project/data/runs/msm_repro/pro-affordability-aft-msm-doctag-20260617-232824/checkpoints/msm",
"result_dir": "/workspace/mats_project/data/runs/msm_repro/pro-affordability-aft-msm-doctag-20260617-232824/checkpoints/msm/eval-suite"
},
{
"label": "swap_afford_doctagc4",
"served": "swap_afford_doctagc4",
"checkpoint": "/workspace/mats_project/data/runs/msm_repro/pro-affordability-aft-msm-doctag-c4-20260618-073215/checkpoints/msm",
"result_dir": "/workspace/mats_project/data/runs/msm_repro/pro-affordability-aft-msm-doctag-c4-20260618-073215/checkpoints/msm/eval-suite"
},
{
"label": "swap_afford_plain",
"served": "swap_afford_plain",
"checkpoint": "/workspace/mats_project/data/runs/msm_repro/pro-affordability-aft-msm-20260612-023024/checkpoints/msm",
"result_dir": "/workspace/mats_project/data/runs/msm_repro/pro-affordability-aft-msm-20260612-023024/checkpoints/msm/eval-suite"
},
{
"label": "swap_america_plain",
"served": "swap_america_plain",
"checkpoint": "/workspace/mats_project/data/runs/msm_repro/pro-america-aft-msm-20260612-025753/checkpoints/msm",
"result_dir": "/workspace/mats_project/data/runs/msm_repro/pro-america-aft-msm-20260612-025753/checkpoints/msm/eval-suite"
}
],
"results": "per arm: <result_dir>/metrics.jsonl and <result_dir>/inspect/**.json"
} |