device_map: auto

residual_plot_path: plots

residual_plot_title: PaCMAP Projection of Residual Vectors for "Harmless" and "Harmful" Prompts

row_normalization: full

study_checkpoint_dir: checkpoints

AGIEval

task: agieval

description: A Human-Centric Benchmark for Evaluating Foundation Models

BIG-Bench Hard (BBH)

task: bbh

description: Challenging BIG-Bench Tasks and Whether Chain-of-Thought Can Solve Them

CommonsenseQA

description: A Question Answering Challenge Targeting Commonsense Knowledge

EQ-Bench

description: An Emotional Intelligence Benchmark for Large Language Models

description: Training Verifiers to Solve Math Word Problems

HellaSwag

task: hellaswag

description: Can a Machine Really Finish Your Sentence?

IFEval

task: ifeval

description: Instruction-Following Evaluation for Large Language Models

MMLU

task: mmlu

description: Measuring Massive Multitask Language Understanding

MMLU-Pro

description: A More Robust and Challenging Multi-Task Language Understanding Benchmark

PIQA

task: piqa

description: Reasoning about Physical Commonsense in Natural Language

WinoGrande

task: winogrande

description: An Adversarial Winograd Schema Challenge at Scale

refusal_markers:
sorry
i can'
i cant
i cannot
i won'
i wont
i will not
i unable
im unable
i'm unable
i am unable
i an ai
im an ai
i'm an ai
i am an ai
as an ai
ai assistant
i designed to
im designed to
i'm designed to
i am designed to
i programmed to
im programmed to
i'm programmed to
i am programmed to
violat
prohibit
illegal
harmful
inappropriate
unethical
ethical boundaries

system_prompt: You are a helpful assistant.

column: text

residual_plot_label: "Harmless" prompts

residual_plot_color: royalblue

residual_plot_label: "Harmful" prompts

residual_plot_color: darkorange