| run: | |
| name: exp1 | |
| output_root: data/runs/eval_suite | |
| runner: h200 | |
| runners: | |
| h200: | |
| tensor_parallel: 1 | |
| gpu_memory_utilization: 0.92 | |
| max_num_seqs: 128 | |
| port: 8000 | |
| two_h100: | |
| tensor_parallel: 2 | |
| gpu_memory_utilization: 0.9 | |
| max_num_seqs: 96 | |
| port: 8000 | |
| auto: | |
| tensor_parallel: auto | |
| gpu_memory_utilization: 0.92 | |
| max_num_seqs: 128 | |
| port: 8000 | |
| model: | |
| id: meta-llama/Llama-3.1-8B | |
| substrate: exp1 | |
| max_model_len: 8192 | |
| max_lora_rank: 128 | |
| max_loras: auto | |
| reasoning_parser: none | |
| enable_thinking: false | |
| chat_template: auto | |
| judge: anthropic/claude-sonnet-4-6 | |
| max_connections: 64 | |
| keep_serving: false | |
| arms: [] | |
| suites: | |
| benign_agentic: | |
| type: inspect | |
| tasks: | |
| - name: benign_agentic | |
| task: why_gen/inspect_tasks/benign_agentic.py@benign_agentic | |
| epochs: 5 | |
| temperature: 1.0 | |
| max_tokens: 4096 | |
| task_args: | |
| tool_format: am_xml | |