{ "anthropic/TrustAIRLab-in-the-wild-jailbreak-prompts/claude-haiku-4-5": { "model_provider_use_case": "anthropic", "dataset_name": "TrustAIRLab-in-the-wild-jailbreak-prompts", "model_name": "claude-haiku-4-5", "accuracy": 0.5505865102639296, "accuracy_per_category": { "true": 0.5505865102639296 }, "fpr": "N/A", "mean_latency": 0.9137335029437388, "latency_ci_95": { "lower": 0.7448164083911873, "upper": 1.0826505974962901, "mean": 0.9137335029437388, "std_dev": 1.0160730479087696, "n": 139 }, "provider": "Anthropic", "model_type": "generalist", "model_developer": "Anthropic", "model_url": "https://platform.claude.com/docs/en/about-claude/models/overview", "cost_info": { "cost_per_1M_input_tokens": 1.0, "cost_per_1M_output_tokens": 5.0, "cost_per_h": "N/A", "cost_source": "Claude API pricing", "cost_additional_info": "", "total_cost": 0.98001 }, "execution_specifications": { "type": "API", "details": "REST API accessed through a CPU3 RunPod instance at US-KS-2." }, "num_samples": 1364 }, "anthropic/TrustAIRLab-in-the-wild-jailbreak-prompts/claude-sonnet-4-5": { "model_provider_use_case": "anthropic", "dataset_name": "TrustAIRLab-in-the-wild-jailbreak-prompts", "model_name": "claude-sonnet-4-5", "accuracy": 0.5285923753665689, "accuracy_per_category": { "true": 0.5285923753665689 }, "fpr": "N/A", "mean_latency": 1.6597283538296925, "latency_ci_95": { "lower": 1.5881466615643636, "upper": 1.7313100460950215, "mean": 1.6597283538296925, "std_dev": 0.43057944151338, "n": 139 }, "provider": "Anthropic", "model_type": "generalist", "model_developer": "Anthropic", "model_url": "https://platform.claude.com/docs/en/about-claude/models/overview", "cost_info": { "cost_per_1M_input_tokens": 3.0, "cost_per_1M_output_tokens": 15.0, "cost_per_h": "N/A", "cost_source": "Claude API pricing", "cost_additional_info": "", "total_cost": 2.9401500000000005 }, "execution_specifications": { "type": "API", "details": "REST API accessed through a CPU3 RunPod instance at US-KS-2." }, "num_samples": 1364 }, "anthropic/TrustAIRLab-in-the-wild-jailbreak-prompts/claude-sonnet-4-6": { "model_provider_use_case": "anthropic", "dataset_name": "TrustAIRLab-in-the-wild-jailbreak-prompts", "model_name": "claude-sonnet-4-6", "accuracy": 0.6466275659824047, "accuracy_per_category": { "true": 0.6466275659824047 }, "fpr": "N/A", "mean_latency": 2.054156614363718, "latency_ci_95": { "lower": 2.0228560722011775, "upper": 2.0854571565262585, "mean": 2.054156614363718, "std_dev": 0.5897975632233993, "n": 1364 }, "provider": "Anthropic", "model_type": "generalist", "model_developer": "Anthropic", "model_url": "https://platform.claude.com/docs/en/about-claude/models/overview", "cost_info": { "cost_per_1M_input_tokens": 3.0, "cost_per_1M_output_tokens": 15.0, "cost_per_h": "N/A", "cost_source": "Claude API pricing", "cost_additional_info": "", "total_cost": 2.930262 }, "execution_specifications": { "type": "API", "details": "REST API accessed through a CPU3 RunPod instance at US-KS-2." }, "num_samples": 1364 }, "anthropic/allenai-wildjailbreak/claude-haiku-4-5": { "model_provider_use_case": "anthropic", "dataset_name": "allenai-wildjailbreak", "model_name": "claude-haiku-4-5", "accuracy": 0.502, "accuracy_per_category": { "adversarial_harmful": 0.502 }, "fpr": 0.12380952380952381, "mean_latency": 0.7089712695298691, "latency_ci_95": { "lower": 0.6671740036241831, "upper": 0.7507685354355551, "mean": 0.7089712695298691, "std_dev": 0.3170209328591228, "n": 221 }, "provider": "Anthropic", "model_type": "generalist", "model_developer": "Anthropic", "model_url": "https://platform.claude.com/docs/en/about-claude/models/overview", "cost_info": { "cost_per_1M_input_tokens": 1.0, "cost_per_1M_output_tokens": 5.0, "cost_per_h": "N/A", "cost_source": "Claude API pricing", "cost_additional_info": "", "total_cost": 0.500946 }, "execution_specifications": { "type": "API", "details": "REST API accessed through a CPU3 RunPod instance at US-KS-2." }, "num_samples": 2210 }, "anthropic/allenai-wildjailbreak/claude-sonnet-4-5": { "model_provider_use_case": "anthropic", "dataset_name": "allenai-wildjailbreak", "model_name": "claude-sonnet-4-5", "accuracy": 0.8385, "accuracy_per_category": { "adversarial_harmful": 0.8385 }, "fpr": 0.24761904761904763, "mean_latency": 1.6692387658546413, "latency_ci_95": { "lower": 1.6105816148829564, "upper": 1.7278959168263262, "mean": 1.6692387658546413, "std_dev": 0.4448985912586268, "n": 221 }, "provider": "Anthropic", "model_type": "generalist", "model_developer": "Anthropic", "model_url": "https://platform.claude.com/docs/en/about-claude/models/overview", "cost_info": { "cost_per_1M_input_tokens": 3.0, "cost_per_1M_output_tokens": 15.0, "cost_per_h": "N/A", "cost_source": "Claude API pricing", "cost_additional_info": "", "total_cost": 1.508748 }, "execution_specifications": { "type": "API", "details": "REST API accessed through a CPU3 RunPod instance at US-KS-2." }, "num_samples": 2210 }, "anthropic/allenai-wildjailbreak/claude-sonnet-4-6": { "model_provider_use_case": "anthropic", "dataset_name": "allenai-wildjailbreak", "model_name": "claude-sonnet-4-6", "accuracy": 0.8245, "accuracy_per_category": { "adversarial_harmful": 0.8245 }, "fpr": 0.0761904761904762, "mean_latency": 2.159039001982676, "latency_ci_95": { "lower": 2.138023675760024, "upper": 2.1800543282053284, "mean": 2.159039001982676, "std_dev": 0.5040529971779389, "n": 2210 }, "provider": "Anthropic", "model_type": "generalist", "model_developer": "Anthropic", "model_url": "https://platform.claude.com/docs/en/about-claude/models/overview", "cost_info": { "cost_per_1M_input_tokens": 3.0, "cost_per_1M_output_tokens": 15.0, "cost_per_h": "N/A", "cost_source": "Claude API pricing", "cost_additional_info": "", "total_cost": 1.4942429999999998 }, "execution_specifications": { "type": "API", "details": "REST API accessed through a CPU3 RunPod instance at US-KS-2." }, "num_samples": 2210 }, "anthropic/centrepourlasecuriteia-BET-jailbreak-dataset/claude-haiku-4-5": { "model_provider_use_case": "anthropic", "dataset_name": "centrepourlasecuriteia-BET-jailbreak-dataset", "model_name": "claude-haiku-4-5", "accuracy": 0.6781246204299769, "accuracy_per_category": { "privacy": 0.5006134969325153, "harm to minors": 0.6658536585365854, "integrity & quality violations": 0.5552855407047388, "illegal activities": 0.7680097680097681, "sexual content": 0.7867469879518072, "physical harm": 0.6479041916167665, "cbrn": 0.8118932038834952, "harmful manipulation": 0.8523409363745498, "cyber": 0.6009852216748769, "self-harm": 0.5863746958637469 }, "fpr": 0.09449760765550239, "mean_latency": 1.1553465698991654, "latency_ci_95": { "lower": 1.112908704224636, "upper": 1.1977844355736949, "mean": 1.1553465698991654, "std_dev": 0.6517207690329253, "n": 906 }, "provider": "Anthropic", "model_type": "generalist", "model_developer": "Anthropic", "model_url": "https://platform.claude.com/docs/en/about-claude/models/overview", "cost_info": { "cost_per_1M_input_tokens": 1.0, "cost_per_1M_output_tokens": 5.0, "cost_per_h": "N/A", "cost_source": "Claude API pricing", "cost_additional_info": "", "total_cost": 3.884634 }, "execution_specifications": { "type": "API", "details": "REST API accessed through a CPU3 RunPod instance at US-KS-2." }, "num_samples": 9069 }, "anthropic/centrepourlasecuriteia-BET-jailbreak-dataset/claude-sonnet-4-5": { "model_provider_use_case": "anthropic", "dataset_name": "centrepourlasecuriteia-BET-jailbreak-dataset", "model_name": "claude-sonnet-4-5", "accuracy": 0.6889347746872343, "accuracy_per_category": { "privacy": 0.6208588957055214, "harm to minors": 0.675609756097561, "integrity & quality violations": 0.6464155528554071, "illegal activities": 0.6190476190476191, "sexual content": 0.7012048192771084, "physical harm": 0.6994011976047905, "cbrn": 0.8531553398058253, "harmful manipulation": 0.7442977190876351, "cyber": 0.6477832512315271, "self-harm": 0.6788321167883211 }, "fpr": 0.2511961722488038, "mean_latency": 1.7826606514174943, "latency_ci_95": { "lower": 1.7386279958730628, "upper": 1.8266933069619258, "mean": 1.7826606514174943, "std_dev": 0.6762120497309152, "n": 906 }, "provider": "Anthropic", "model_type": "generalist", "model_developer": "Anthropic", "model_url": "https://platform.claude.com/docs/en/about-claude/models/overview", "cost_info": { "cost_per_1M_input_tokens": 3.0, "cost_per_1M_output_tokens": 15.0, "cost_per_h": "N/A", "cost_source": "Claude API pricing", "cost_additional_info": "", "total_cost": 11.577642 }, "execution_specifications": { "type": "API", "details": "REST API accessed through a CPU3 RunPod instance at US-KS-2." }, "num_samples": 9069 }, "anthropic/centrepourlasecuriteia-BET-jailbreak-dataset/claude-sonnet-4-6": { "model_provider_use_case": "anthropic", "dataset_name": "centrepourlasecuriteia-BET-jailbreak-dataset", "model_name": "claude-sonnet-4-6", "accuracy": 0.8256590936702709, "accuracy_per_category": { "privacy": 0.596319018404908, "harm to minors": 0.608058608058608, "integrity & quality violations": 0.8517618469015796, "illegal activities": 0.884004884004884, "sexual content": 0.9349397590361446, "physical harm": 0.8083832335329342, "cbrn": 0.9575242718446602, "harmful manipulation": 0.9087635054021609, "cyber": 0.8717632552404438, "self-harm": 0.8309002433090025 }, "fpr": 0.17942583732057416, "mean_latency": 1.8611644692230056, "latency_ci_95": { "lower": 1.845762867540732, "upper": 1.8765660709052792, "mean": 1.8611644692230056, "std_dev": 0.7482412105097435, "n": 9067 }, "provider": "Anthropic", "model_type": "generalist", "model_developer": "Anthropic", "model_url": "https://platform.claude.com/docs/en/about-claude/models/overview", "cost_info": { "cost_per_1M_input_tokens": 3.0, "cost_per_1M_output_tokens": 15.0, "cost_per_h": "N/A", "cost_source": "Claude API pricing", "cost_additional_info": "", "total_cost": 11.51268 }, "execution_specifications": { "type": "API", "details": "REST API accessed through a CPU3 RunPod instance at US-KS-2." }, "num_samples": 9067 }, "anthropic/centrepourlasecuriteia-jailbreak-dataset/claude-haiku-4-5": { "model_provider_use_case": "anthropic", "dataset_name": "centrepourlasecuriteia-jailbreak-dataset", "model_name": "claude-haiku-4-5", "accuracy": 0.6204830216088615, "accuracy_per_category": { "integrity & quality violations": 0.42536115569823435, "physical harm": 0.7161654135338346, "harm to minors": 0.7843137254901961, "illegal activities": 0.6697761194029851, "cyber": 0.6860902255639098, "privacy": 0.5570093457943925, "sexual content": 0.6841121495327103, "cbrn": 0.7313432835820896, "self-harm": 0.4632768361581921, "harmful manipulation": 0.4953271028037383 }, "fpr": 0.017797552836484983, "mean_latency": 0.8135531477630138, "latency_ci_95": { "lower": 0.7854269088842167, "upper": 0.8416793866418109, "mean": 0.8135531477630138, "std_dev": 0.3630325586570585, "n": 640 }, "provider": "Anthropic", "model_type": "generalist", "model_developer": "Anthropic", "model_url": "https://platform.claude.com/docs/en/about-claude/models/overview", "cost_info": { "cost_per_1M_input_tokens": 1.0, "cost_per_1M_output_tokens": 5.0, "cost_per_h": "N/A", "cost_source": "Claude API pricing", "cost_additional_info": "", "total_cost": 3.2963259999999996 }, "execution_specifications": { "type": "API", "details": "REST API accessed through a CPU3 RunPod instance at US-KS-2." }, "num_samples": 6406, "accuracy_per_technique": { "ascii_art": 0.06774193548387097, "fsh": 0.7645161290322581, "adversarial_suffixes": 0.7661290322580645, "tokenbreak": 0.7806451612903226, "structural_obfuscation": 0.8096774193548387, "cognitive_psychological": 0.49516129032258066, "low_resource_language": 0.4113345521023766, "encoding_cyphering": 0.6548387096774193, "dap": 0.8096774193548387 } }, "anthropic/centrepourlasecuriteia-jailbreak-dataset/claude-sonnet-4-5": { "model_provider_use_case": "anthropic", "dataset_name": "centrepourlasecuriteia-jailbreak-dataset", "model_name": "claude-sonnet-4-5", "accuracy": 0.7937170873433812, "accuracy_per_category": { "integrity & quality violations": 0.6581059390048154, "physical harm": 0.8515037593984962, "harm to minors": 0.8545751633986928, "illegal activities": 0.8115671641791045, "cyber": 0.8195488721804511, "privacy": 0.7626168224299066, "sexual content": 0.8261682242990654, "cbrn": 0.8824626865671642, "self-harm": 0.7382297551789078, "harmful manipulation": 0.7457943925233644 }, "fpr": 0.19577308120133483, "mean_latency": 1.8916679993271828, "latency_ci_95": { "lower": 1.830309470788241, "upper": 1.9530265278661245, "mean": 1.8916679993271828, "std_dev": 0.791970220651017, "n": 640 }, "provider": "Anthropic", "model_type": "generalist", "model_developer": "Anthropic", "model_url": "https://platform.claude.com/docs/en/about-claude/models/overview", "cost_info": { "cost_per_1M_input_tokens": 3.0, "cost_per_1M_output_tokens": 15.0, "cost_per_h": "N/A", "cost_source": "Claude API pricing", "cost_additional_info": "", "total_cost": 9.827823 }, "execution_specifications": { "type": "API", "details": "REST API accessed through a CPU3 RunPod instance at US-KS-2." }, "num_samples": 6406, "accuracy_per_technique": { "ascii_art": 0.36451612903225805, "fsh": 0.8419354838709677, "adversarial_suffixes": 0.896774193548387, "tokenbreak": 0.8774193548387097, "structural_obfuscation": 0.8806451612903226, "cognitive_psychological": 0.6870967741935484, "low_resource_language": 0.7659963436928702, "encoding_cyphering": 0.9903225806451613, "dap": 0.8354838709677419 } }, "anthropic/centrepourlasecuriteia-jailbreak-dataset/claude-sonnet-4-6": { "model_provider_use_case": "anthropic", "dataset_name": "centrepourlasecuriteia-jailbreak-dataset", "model_name": "claude-sonnet-4-6", "accuracy": 0.7457781005992373, "accuracy_per_category": { "integrity & quality violations": 0.5778491171749599, "physical harm": 0.825187969924812, "harm to minors": 0.8398692810457516, "illegal activities": 0.8097014925373134, "cyber": 0.8270676691729323, "privacy": 0.7233644859813084, "sexual content": 0.7532710280373832, "cbrn": 0.8638059701492538, "self-harm": 0.5969868173258004, "harmful manipulation": 0.6542056074766355 }, "fpr": 0.1668520578420467, "mean_latency": 1.8029755075536888, "latency_ci_95": { "lower": 1.784683509708755, "upper": 1.8212675053986227, "mean": 1.8029755075536888, "std_dev": 0.7469620493978066, "n": 6406 }, "provider": "Anthropic", "model_type": "generalist", "model_developer": "Anthropic", "model_url": "https://platform.claude.com/docs/en/about-claude/models/overview", "cost_info": { "cost_per_1M_input_tokens": 3.0, "cost_per_1M_output_tokens": 15.0, "cost_per_h": "N/A", "cost_source": "Claude API pricing", "cost_additional_info": "", "total_cost": 9.830495999999998 }, "execution_specifications": { "type": "API", "details": "REST API accessed through a CPU3 RunPod instance at US-KS-2." }, "num_samples": 6406, "accuracy_per_technique": { "ascii_art": 0.4258064516129032, "fsh": 0.7467741935483871, "adversarial_suffixes": 0.7693548387096775, "tokenbreak": 0.8, "structural_obfuscation": 0.885483870967742, "cognitive_psychological": 0.646774193548387, "low_resource_language": 0.623400365630713, "encoding_cyphering": 0.9790322580645161, "dap": 0.8209677419354838 } }, "anthropic/deepset-prompt-injections/claude-haiku-4-5": { "model_provider_use_case": "anthropic", "dataset_name": "deepset-prompt-injections", "model_name": "claude-haiku-4-5", "accuracy": 0.049429657794676805, "accuracy_per_category": { "1": 0.049429657794676805 }, "fpr": 0.0, "mean_latency": 0.6440598788084807, "latency_ci_95": { "lower": 0.5803725244390934, "upper": 0.707747233177868, "mean": 0.6440598788084807, "std_dev": 0.2387778386828749, "n": 54 }, "provider": "Anthropic", "model_type": "generalist", "model_developer": "Anthropic", "model_url": "https://platform.claude.com/docs/en/about-claude/models/overview", "cost_info": { "cost_per_1M_input_tokens": 1.0, "cost_per_1M_output_tokens": 5.0, "cost_per_h": "N/A", "cost_source": "Claude API pricing", "cost_additional_info": "", "total_cost": 0.065629 }, "execution_specifications": { "type": "API", "details": "REST API accessed through a CPU3 RunPod instance at US-KS-2." }, "num_samples": 662 }, "anthropic/deepset-prompt-injections/claude-sonnet-4-5": { "model_provider_use_case": "anthropic", "dataset_name": "deepset-prompt-injections", "model_name": "claude-sonnet-4-5", "accuracy": 0.4600760456273764, "accuracy_per_category": { "1": 0.4600760456273764 }, "fpr": 0.040100250626566414, "mean_latency": 1.62129075438888, "latency_ci_95": { "lower": 1.521633846924404, "upper": 1.720947661853356, "mean": 1.62129075438888, "std_dev": 0.37363557035468276, "n": 54 }, "provider": "Anthropic", "model_type": "generalist", "model_developer": "Anthropic", "model_url": "https://platform.claude.com/docs/en/about-claude/models/overview", "cost_info": { "cost_per_1M_input_tokens": 3.0, "cost_per_1M_output_tokens": 15.0, "cost_per_h": "N/A", "cost_source": "Claude API pricing", "cost_additional_info": "", "total_cost": 0.20183700000000002 }, "execution_specifications": { "type": "API", "details": "REST API accessed through a CPU3 RunPod instance at US-KS-2." }, "num_samples": 662 }, "anthropic/deepset-prompt-injections/claude-sonnet-4-6": { "model_provider_use_case": "anthropic", "dataset_name": "deepset-prompt-injections", "model_name": "claude-sonnet-4-6", "accuracy": 0.19011406844106463, "accuracy_per_category": { "1": 0.19011406844106463 }, "fpr": 0.007518796992481203, "mean_latency": 2.1657514932292465, "latency_ci_95": { "lower": 2.1316342046263106, "upper": 2.1998687818321825, "mean": 2.1657514932292465, "std_dev": 0.447865317971721, "n": 662 }, "provider": "Anthropic", "model_type": "generalist", "model_developer": "Anthropic", "model_url": "https://platform.claude.com/docs/en/about-claude/models/overview", "cost_info": { "cost_per_1M_input_tokens": 3.0, "cost_per_1M_output_tokens": 15.0, "cost_per_h": "N/A", "cost_source": "Claude API pricing", "cost_additional_info": "", "total_cost": 0.198753 }, "execution_specifications": { "type": "API", "details": "REST API accessed through a CPU3 RunPod instance at US-KS-2." }, "num_samples": 662 }, "anthropic/local_datasets-malignant/claude-haiku-4-5": { "model_provider_use_case": "anthropic", "dataset_name": "local_datasets-malignant", "model_name": "claude-haiku-4-5", "accuracy": 0.5979899497487438, "accuracy_per_category": { "jailbreak": 0.5979899497487438 }, "fpr": 0.0021945866861741038, "mean_latency": 0.663169888001454, "latency_ci_95": { "lower": 0.6230578682368119, "upper": 0.7032819077660962, "mean": 0.663169888001454, "std_dev": 0.2572450358198157, "n": 158 }, "provider": "Anthropic", "model_type": "generalist", "model_developer": "Anthropic", "model_url": "https://platform.claude.com/docs/en/about-claude/models/overview", "cost_info": { "cost_per_1M_input_tokens": 1.0, "cost_per_1M_output_tokens": 5.0, "cost_per_h": "N/A", "cost_source": "Claude API pricing", "cost_additional_info": "", "total_cost": 0.20477 }, "execution_specifications": { "type": "API", "details": "REST API accessed through a CPU3 RunPod instance at US-KS-2." }, "num_samples": 1566 }, "anthropic/local_datasets-malignant/claude-sonnet-4-5": { "model_provider_use_case": "anthropic", "dataset_name": "local_datasets-malignant", "model_name": "claude-sonnet-4-5", "accuracy": 0.6381909547738693, "accuracy_per_category": { "jailbreak": 0.6381909547738693 }, "fpr": 0.04023408924652524, "mean_latency": 1.6170983933195282, "latency_ci_95": { "lower": 1.5469866639673255, "upper": 1.6872101226717309, "mean": 1.6170983933195282, "std_dev": 0.4496381492236606, "n": 158 }, "provider": "Anthropic", "model_type": "generalist", "model_developer": "Anthropic", "model_url": "https://platform.claude.com/docs/en/about-claude/models/overview", "cost_info": { "cost_per_1M_input_tokens": 3.0, "cost_per_1M_output_tokens": 15.0, "cost_per_h": "N/A", "cost_source": "Claude API pricing", "cost_additional_info": "", "total_cost": 0.67989 }, "execution_specifications": { "type": "API", "details": "REST API accessed through a CPU3 RunPod instance at US-KS-2." }, "num_samples": 1566 }, "anthropic/local_datasets-malignant/claude-sonnet-4-6": { "model_provider_use_case": "anthropic", "dataset_name": "local_datasets-malignant", "model_name": "claude-sonnet-4-6", "accuracy": 0.7286432160804021, "accuracy_per_category": { "jailbreak": 0.7286432160804021 }, "fpr": 0.001463057790782736, "mean_latency": 2.164280016303519, "latency_ci_95": { "lower": 2.1207516900773955, "upper": 2.207808342529643, "mean": 2.164280016303519, "std_dev": 0.8788439659742222, "n": 1566 }, "provider": "Anthropic", "model_type": "generalist", "model_developer": "Anthropic", "model_url": "https://platform.claude.com/docs/en/about-claude/models/overview", "cost_info": { "cost_per_1M_input_tokens": 3.0, "cost_per_1M_output_tokens": 15.0, "cost_per_h": "N/A", "cost_source": "Claude API pricing", "cost_additional_info": "", "total_cost": 0.616548 }, "execution_specifications": { "type": "API", "details": "REST API accessed through a CPU3 RunPod instance at US-KS-2." }, "num_samples": 1566 }, "google/TrustAIRLab-in-the-wild-jailbreak-prompts/gemini-2.5-flash": { "model_provider_use_case": "google", "dataset_name": "TrustAIRLab-in-the-wild-jailbreak-prompts", "model_name": "gemini-2.5-flash", "accuracy": 0.7573313782991202, "accuracy_per_category": { "true": 0.7573313782991202 }, "fpr": "N/A", "mean_latency": 2.5284244842666515, "latency_ci_95": { "lower": 2.252913864573483, "upper": 2.80393510395982, "mean": 2.5284244842666515, "std_dev": 1.6572562760708647, "n": 139 }, "provider": "GoogleAIStudio", "model_type": "generalist", "model_developer": "Google", "model_url": "https://ai.google.dev/gemini-api/docs/models#gemini-2.5-flash", "cost_info": { "cost_per_1M_input_tokens": 0.3, "cost_per_1M_output_tokens": 2.5, "cost_per_h": "N/A", "cost_source": "Gemini Developer API pricing", "cost_additional_info": "", "total_cost": 1.4500211 }, "execution_specifications": { "type": "API", "details": "REST API accessed through a CPU3 RunPod instance at US-KS-2." }, "num_samples": 1364 }, "google/allenai-wildjailbreak/gemini-2.5-flash": { "model_provider_use_case": "google", "dataset_name": "allenai-wildjailbreak", "model_name": "gemini-2.5-flash", "accuracy": 0.8575, "accuracy_per_category": { "adversarial_harmful": 0.8575 }, "fpr": 0.2, "mean_latency": 2.9715932969054486, "latency_ci_95": { "lower": 2.735281703061325, "upper": 3.2079048907495724, "mean": 2.9715932969054486, "std_dev": 1.792359387691404, "n": 221 }, "provider": "GoogleAIStudio", "model_type": "generalist", "model_developer": "Google", "model_url": "https://ai.google.dev/gemini-api/docs/models#gemini-2.5-flash", "cost_info": { "cost_per_1M_input_tokens": 0.3, "cost_per_1M_output_tokens": 2.5, "cost_per_h": "N/A", "cost_source": "Gemini Developer API pricing", "cost_additional_info": "", "total_cost": 2.3660427 }, "execution_specifications": { "type": "API", "details": "REST API accessed through a CPU3 RunPod instance at US-KS-2." }, "num_samples": 2210 }, "google/centrepourlasecuriteia-BET-jailbreak-dataset/gemini-2.5-flash": { "model_provider_use_case": "google", "dataset_name": "centrepourlasecuriteia-BET-jailbreak-dataset", "model_name": "gemini-2.5-flash", "accuracy": 0.9306449653832138, "accuracy_per_category": { "privacy": 0.9325153374233128, "harm to minors": 0.7646341463414634, "integrity & quality violations": 0.9720534629404617, "illegal activities": 0.9584859584859585, "sexual content": 0.9216867469879518, "physical harm": 0.9664670658682635, "cbrn": 0.9381067961165048, "harmful manipulation": 0.9375750300120048, "cyber": 0.9495073891625616, "self-harm": 0.9647201946472019 }, "fpr": 0.14832535885167464, "mean_latency": 2.9566942458647505, "latency_ci_95": { "lower": 2.6249056582715973, "upper": 3.2884828334579037, "mean": 2.9566942458647505, "std_dev": 5.095296618376791, "n": 906 }, "provider": "GoogleAIStudio", "model_type": "generalist", "model_developer": "Google", "model_url": "https://ai.google.dev/gemini-api/docs/models#gemini-2.5-flash", "cost_info": { "cost_per_1M_input_tokens": 0.3, "cost_per_1M_output_tokens": 2.5, "cost_per_h": "N/A", "cost_source": "Gemini Developer API pricing", "cost_additional_info": "", "total_cost": 13.1941337 }, "execution_specifications": { "type": "API", "details": "REST API accessed through a CPU3 RunPod instance at US-KS-2." }, "num_samples": 9069 }, "google/centrepourlasecuriteia-jailbreak-dataset/gemini-2.5-flash": { "model_provider_use_case": "google", "dataset_name": "centrepourlasecuriteia-jailbreak-dataset", "model_name": "gemini-2.5-flash", "accuracy": 0.81332849101144, "accuracy_per_category": { "integrity & quality violations": 0.7399678972712681, "physical harm": 0.8778195488721805, "harm to minors": 0.6617647058823529, "illegal activities": 0.8544776119402985, "cyber": 0.8665413533834586, "privacy": 0.8672897196261682, "sexual content": 0.7906542056074767, "cbrn": 0.8731343283582089, "self-harm": 0.8248587570621468, "harmful manipulation": 0.811214953271028 }, "fpr": 0.11457174638487208, "mean_latency": 2.473413760960102, "latency_ci_95": { "lower": 2.1817201597964475, "upper": 2.7651073621237563, "mean": 2.473413760960102, "std_dev": 3.7649639125464867, "n": 640 }, "provider": "GoogleAIStudio", "model_type": "generalist", "model_developer": "Google", "model_url": "https://ai.google.dev/gemini-api/docs/models#gemini-2.5-flash", "cost_info": { "cost_per_1M_input_tokens": 0.3, "cost_per_1M_output_tokens": 2.5, "cost_per_h": "N/A", "cost_source": "Gemini Developer API pricing", "cost_additional_info": "", "total_cost": 6.445347299999999 }, "execution_specifications": { "type": "API", "details": "REST API accessed through a CPU3 RunPod instance at US-KS-2." }, "num_samples": 6406, "accuracy_per_technique": { "ascii_art": 0.18225806451612903, "fsh": 0.9338709677419355, "adversarial_suffixes": 0.8903225806451613, "tokenbreak": 0.8838709677419355, "structural_obfuscation": 0.9403225806451613, "cognitive_psychological": 0.8064516129032258, "low_resource_language": 0.8354661791590493, "encoding_cyphering": 0.9048387096774193, "dap": 0.9451612903225807 } }, "google/deepset-prompt-injections/gemini-2.5-flash": { "model_provider_use_case": "google", "dataset_name": "deepset-prompt-injections", "model_name": "gemini-2.5-flash", "accuracy": 0.45627376425855515, "accuracy_per_category": { "1": 0.45627376425855515 }, "fpr": 0.06015037593984962, "mean_latency": 1.556288445437396, "latency_ci_95": { "lower": 1.2516525754371877, "upper": 1.8609243154376043, "mean": 1.556288445437396, "std_dev": 1.14214659007552, "n": 54 }, "provider": "GoogleAIStudio", "model_type": "generalist", "model_developer": "Google", "model_url": "https://ai.google.dev/gemini-api/docs/models#gemini-2.5-flash", "cost_info": { "cost_per_1M_input_tokens": 0.3, "cost_per_1M_output_tokens": 2.5, "cost_per_h": "N/A", "cost_source": "Gemini Developer API pricing", "cost_additional_info": "", "total_cost": 0.356095 }, "execution_specifications": { "type": "API", "details": "REST API accessed through a CPU3 RunPod instance at US-KS-2." }, "num_samples": 662 }, "google/local_datasets-malignant/gemini-2.5-flash": { "model_provider_use_case": "google", "dataset_name": "local_datasets-malignant", "model_name": "gemini-2.5-flash", "accuracy": 0.8944723618090452, "accuracy_per_category": { "jailbreak": 0.8944723618090452 }, "fpr": 0.016093635698610095, "mean_latency": 1.1865572974651675, "latency_ci_95": { "lower": 1.0899337274497258, "upper": 1.283180867480609, "mean": 1.1865572974651675, "std_dev": 0.619662980710104, "n": 158 }, "provider": "GoogleAIStudio", "model_type": "generalist", "model_developer": "Google", "model_url": "https://ai.google.dev/gemini-api/docs/models#gemini-2.5-flash", "cost_info": { "cost_per_1M_input_tokens": 0.3, "cost_per_1M_output_tokens": 2.5, "cost_per_h": "N/A", "cost_source": "Gemini Developer API pricing", "cost_additional_info": "", "total_cost": 0.5181074999999999 }, "execution_specifications": { "type": "API", "details": "REST API accessed through a CPU3 RunPod instance at US-KS-2." }, "num_samples": 1566 }, "lakera/TrustAIRLab-in-the-wild-jailbreak-prompts/lakera-guard_default": { "model_provider_use_case": "lakera", "dataset_name": "TrustAIRLab-in-the-wild-jailbreak-prompts", "model_name": "lakera-guard_default", "accuracy": 0.9750733137829912, "accuracy_per_category": { "true": 0.9750733137829912 }, "fpr": "N/A", "mean_latency": 0.22058862061809292, "latency_ci_95": { "lower": 0.2118761152700835, "upper": 0.22930112596610236, "mean": 0.22058862061809292, "std_dev": 0.05240761384940411, "n": 139 }, "provider": "Lakera", "model_type": "specialized", "model_developer": "Lakera", "model_url": "https://www.lakera.ai/lakera-guard", "cost_info": { "cost_per_1M_input_tokens": 0.0, "cost_per_1M_output_tokens": 0.0, "cost_per_h": "N/A", "cost_source": "Lakera API pricing", "cost_additional_info": "Free for 10,000 API requests/month; Allows prompt size up to 8,000 tokens per request.", "total_cost": 0.0 }, "execution_specifications": { "type": "API", "details": "This is the LakeraGuard API with the default policy. REST API accessed through a CPU3 RunPod instance at US-KS-2." }, "num_samples": 1364 }, "lakera/allenai-wildjailbreak/lakera-guard_default": { "model_provider_use_case": "lakera", "dataset_name": "allenai-wildjailbreak", "model_name": "lakera-guard_default", "accuracy": 0.9895, "accuracy_per_category": { "adversarial_harmful": 0.9895 }, "fpr": 0.9619047619047619, "mean_latency": 0.1992312178892248, "latency_ci_95": { "lower": 0.1886028728160705, "upper": 0.2098595629623791, "mean": 0.1992312178892248, "std_dev": 0.08061311659578355, "n": 221 }, "provider": "Lakera", "model_type": "specialized", "model_developer": "Lakera", "model_url": "https://www.lakera.ai/lakera-guard", "cost_info": { "cost_per_1M_input_tokens": 0.0, "cost_per_1M_output_tokens": 0.0, "cost_per_h": "N/A", "cost_source": "Lakera API pricing", "cost_additional_info": "Free for 10,000 API requests/month; Allows prompt size up to 8,000 tokens per request.", "total_cost": 0.0 }, "execution_specifications": { "type": "API", "details": "This is the LakeraGuard API with the default policy. REST API accessed through a CPU3 RunPod instance at US-KS-2." }, "num_samples": 2210 }, "lakera/centrepourlasecuriteia-BET-jailbreak-dataset/lakera-guard_default": { "model_provider_use_case": "lakera", "dataset_name": "centrepourlasecuriteia-BET-jailbreak-dataset", "model_name": "lakera-guard_default", "accuracy": 0.9883396088910482, "accuracy_per_category": { "privacy": 0.9877300613496932, "harm to minors": 0.9878048780487805, "integrity & quality violations": 0.9987849331713244, "illegal activities": 0.9938949938949939, "sexual content": 0.9915662650602409, "physical harm": 0.9904191616766467, "cbrn": 0.9987864077669902, "harmful manipulation": 0.9975990396158463, "cyber": 0.9729064039408867, "self-harm": 0.9635036496350365 }, "fpr": 0.9593301435406698, "mean_latency": 0.21306764158455216, "latency_ci_95": { "lower": 0.21107435757853343, "upper": 0.21506092559057088, "mean": 0.21306764158455216, "std_dev": 0.030610978300994848, "n": 906 }, "provider": "Lakera", "model_type": "specialized", "model_developer": "Lakera", "model_url": "https://www.lakera.ai/lakera-guard", "cost_info": { "cost_per_1M_input_tokens": 0.0, "cost_per_1M_output_tokens": 0.0, "cost_per_h": "N/A", "cost_source": "Lakera API pricing", "cost_additional_info": "Free for 10,000 API requests/month; Allows prompt size up to 8,000 tokens per request.", "total_cost": 0.0 }, "execution_specifications": { "type": "API", "details": "This is the LakeraGuard API with the default policy. REST API accessed through a CPU3 RunPod instance at US-KS-2." }, "num_samples": 9069 }, "lakera/centrepourlasecuriteia-jailbreak-dataset/lakera-guard_default": { "model_provider_use_case": "lakera", "dataset_name": "centrepourlasecuriteia-jailbreak-dataset", "model_name": "lakera-guard_default", "accuracy": 0.8574541492645723, "accuracy_per_category": { "integrity & quality violations": 0.8138041733547352, "physical harm": 0.881578947368421, "harm to minors": 0.9035947712418301, "illegal activities": 0.8582089552238806, "cyber": 0.8703007518796992, "privacy": 0.8448598130841122, "sexual content": 0.897196261682243, "cbrn": 0.8861940298507462, "self-harm": 0.7777777777777778, "harmful manipulation": 0.8411214953271028 }, "fpr": 0.5361512791991101, "mean_latency": 0.22953894957900048, "latency_ci_95": { "lower": 0.2257557958110894, "upper": 0.23332210334691156, "mean": 0.22953894957900048, "std_dev": 0.04883013324590635, "n": 640 }, "provider": "Lakera", "model_type": "specialized", "model_developer": "Lakera", "model_url": "https://www.lakera.ai/lakera-guard", "cost_info": { "cost_per_1M_input_tokens": 0.0, "cost_per_1M_output_tokens": 0.0, "cost_per_h": "N/A", "cost_source": "Lakera API pricing", "cost_additional_info": "Free for 10,000 API requests/month; Allows prompt size up to 8,000 tokens per request.", "total_cost": 0.0 }, "execution_specifications": { "type": "API", "details": "This is the LakeraGuard API with the default policy. REST API accessed through a CPU3 RunPod instance at US-KS-2." }, "num_samples": 6406, "accuracy_per_technique": { "ascii_art": 0.4306451612903226, "fsh": 0.9258064516129032, "adversarial_suffixes": 0.9241935483870968, "tokenbreak": 0.9435483870967742, "structural_obfuscation": 1.0, "cognitive_psychological": 0.9919354838709677, "low_resource_language": 0.8318098720292505, "encoding_cyphering": 0.75, "dap": 0.9161290322580645 } }, "lakera/deepset-prompt-injections/lakera-guard_default": { "model_provider_use_case": "lakera", "dataset_name": "deepset-prompt-injections", "model_name": "lakera-guard_default", "accuracy": 0.9163498098859315, "accuracy_per_category": { "1": 0.9163498098859315 }, "fpr": 0.16040100250626566, "mean_latency": 0.18630617636221428, "latency_ci_95": { "lower": 0.17828717125478727, "upper": 0.19432518146964128, "mean": 0.18630617636221428, "std_dev": 0.030065006262196495, "n": 54 }, "provider": "Lakera", "model_type": "specialized", "model_developer": "Lakera", "model_url": "https://www.lakera.ai/lakera-guard", "cost_info": { "cost_per_1M_input_tokens": 0.0, "cost_per_1M_output_tokens": 0.0, "cost_per_h": "N/A", "cost_source": "Lakera API pricing", "cost_additional_info": "Free for 10,000 API requests/month; Allows prompt size up to 8,000 tokens per request.", "total_cost": 0.0 }, "execution_specifications": { "type": "API", "details": "This is the LakeraGuard API with the default policy. REST API accessed through a CPU3 RunPod instance at US-KS-2." }, "num_samples": 662 }, "lakera/local_datasets-malignant/lakera-guard_default": { "model_provider_use_case": "lakera", "dataset_name": "local_datasets-malignant", "model_name": "lakera-guard_default", "accuracy": 0.9949748743718593, "accuracy_per_category": { "jailbreak": 0.9949748743718593 }, "fpr": 0.12801755669348938, "mean_latency": 0.19277338589294046, "latency_ci_95": { "lower": 0.1881992782689706, "upper": 0.19734749351691033, "mean": 0.19277338589294046, "std_dev": 0.029334510864222972, "n": 158 }, "provider": "Lakera", "model_type": "specialized", "model_developer": "Lakera", "model_url": "https://www.lakera.ai/lakera-guard", "cost_info": { "cost_per_1M_input_tokens": 0.0, "cost_per_1M_output_tokens": 0.0, "cost_per_h": "N/A", "cost_source": "Lakera API pricing", "cost_additional_info": "Free for 10,000 API requests/month; Allows prompt size up to 8,000 tokens per request.", "total_cost": 0.0 }, "execution_specifications": { "type": "API", "details": "This is the LakeraGuard API with the default policy. REST API accessed through a CPU3 RunPod instance at US-KS-2." }, "num_samples": 1566 }, "leolee99/TrustAIRLab-in-the-wild-jailbreak-prompts/piguard": { "model_provider_use_case": "leolee99", "dataset_name": "TrustAIRLab-in-the-wild-jailbreak-prompts", "model_name": "piguard", "accuracy": 0.9178885630498533, "accuracy_per_category": { "true": 0.9178885630498533 }, "fpr": "N/A", "mean_latency": 1.114157457505503, "latency_ci_95": { "lower": 0.7565345623461532, "upper": 1.471780352664853, "mean": 1.114157457505503, "std_dev": 6.738704749028581, "n": 1364 }, "provider": "RunPod", "model_type": "specialized", "model_developer": "Leolee99", "model_url": "https://huggingface.co/leolee99/PIGuard", "cost_info": { "cost_per_1M_input_tokens": 1.2566281500806542, "cost_per_1M_output_tokens": 0.0, "cost_per_h": 2.39, "cost_source": "RunPod", "cost_additional_info": "Output token cost is disregarded. The cost per 1M input tokens is estimated as total_cost * (1,000,000 / total_input_tokens).", "total_cost": 1.0089190958804555 }, "execution_specifications": { "type": "Local", "details": "This model was ran on an H100 (80GB PCIe) on RunPod using the transformers Library." }, "num_samples": 1364 }, "leolee99/allenai-wildjailbreak/piguard": { "model_provider_use_case": "leolee99", "dataset_name": "allenai-wildjailbreak", "model_name": "piguard", "accuracy": 0.607, "accuracy_per_category": { "adversarial_harmful": 0.607 }, "fpr": 0.36666666666666664, "mean_latency": 0.026974777614369113, "latency_ci_95": { "lower": 0.026758299870457716, "upper": 0.02719125535828051, "mean": 0.026974777614369113, "std_dev": 0.005192222784685722, "n": 2210 }, "provider": "RunPod", "model_type": "specialized", "model_developer": "Leolee99", "model_url": "https://huggingface.co/leolee99/PIGuard", "cost_info": { "cost_per_1M_input_tokens": 0.12031897954595035, "cost_per_1M_output_tokens": 0.0, "cost_per_h": 2.39, "cost_source": "RunPod", "cost_additional_info": "Output token cost is disregarded. The cost per 1M input tokens is estimated as total_cost * (1,000,000 / total_input_tokens).", "total_cost": 0.03957724385592672 }, "execution_specifications": { "type": "Local", "details": "This model was ran on an H100 (80GB PCIe) on RunPod using the transformers Library." }, "num_samples": 2210 }, "leolee99/centrepourlasecuriteia-BET-jailbreak-dataset/piguard": { "model_provider_use_case": "leolee99", "dataset_name": "centrepourlasecuriteia-BET-jailbreak-dataset", "model_name": "piguard", "accuracy": 0.8474432163245476, "accuracy_per_category": { "privacy": 0.894478527607362, "harm to minors": 0.8390243902439024, "integrity & quality violations": 0.9550425273390036, "illegal activities": 0.8803418803418803, "sexual content": 0.8361445783132531, "physical harm": 0.7736526946107785, "cbrn": 0.8216019417475728, "harmful manipulation": 0.8679471788715486, "cyber": 0.8399014778325123, "self-harm": 0.767639902676399 }, "fpr": 0.8660287081339713, "mean_latency": 0.030311609105825137, "latency_ci_95": { "lower": 0.02988371862418684, "upper": 0.030739499587463432, "mean": 0.030311609105825137, "std_dev": 0.020790084541553493, "n": 9069 }, "provider": "RunPod", "model_type": "specialized", "model_developer": "Leolee99", "model_url": "https://huggingface.co/leolee99/PIGuard", "cost_info": { "cost_per_1M_input_tokens": 0.062324684860117927, "cost_per_1M_output_tokens": 0.0, "cost_per_h": 2.39, "cost_source": "RunPod", "cost_additional_info": "Output token cost is disregarded. The cost per 1M input tokens is estimated as total_cost * (1,000,000 / total_input_tokens).", "total_cost": 0.18250038870109453 }, "execution_specifications": { "type": "Local", "details": "This model was ran on an H100 (80GB PCIe) on RunPod using the transformers Library." }, "num_samples": 9069 }, "leolee99/centrepourlasecuriteia-jailbreak-dataset/piguard": { "model_provider_use_case": "leolee99", "dataset_name": "centrepourlasecuriteia-jailbreak-dataset", "model_name": "piguard", "accuracy": 0.5349555111676049, "accuracy_per_category": { "integrity & quality violations": 0.5971107544141252, "physical harm": 0.5037593984962406, "harm to minors": 0.5261437908496732, "illegal activities": 0.5223880597014925, "cyber": 0.5582706766917294, "privacy": 0.5401869158878505, "sexual content": 0.502803738317757, "cbrn": 0.5055970149253731, "self-harm": 0.5329566854990584, "harmful manipulation": 0.5514018691588785 }, "fpr": 0.5228031145717463, "mean_latency": 0.030191327868273138, "latency_ci_95": { "lower": 0.029378275403524383, "upper": 0.031004380333021893, "mean": 0.030191327868273138, "std_dev": 0.03320136709423847, "n": 6406 }, "provider": "RunPod", "model_type": "specialized", "model_developer": "Leolee99", "model_url": "https://huggingface.co/leolee99/PIGuard", "cost_info": { "cost_per_1M_input_tokens": 0.04857182401507941, "cost_per_1M_output_tokens": 0.0, "cost_per_h": 2.39, "cost_source": "RunPod", "cost_additional_info": "Output token cost is disregarded. The cost per 1M input tokens is estimated as total_cost * (1,000,000 / total_input_tokens).", "total_cost": 0.12839985964298248 }, "execution_specifications": { "type": "Local", "details": "This model was ran on an H100 (80GB PCIe) on RunPod using the transformers Library." }, "num_samples": 6406, "accuracy_per_technique": { "ascii_art": 0.9, "fsh": 0.056451612903225805, "adversarial_suffixes": 0.4854838709677419, "tokenbreak": 0.3903225806451613, "structural_obfuscation": 1.0, "cognitive_psychological": 0.9129032258064517, "low_resource_language": 0.1663619744058501, "encoding_cyphering": 0.7951612903225806, "dap": 0.06451612903225806 } }, "leolee99/deepset-prompt-injections/piguard": { "model_provider_use_case": "leolee99", "dataset_name": "deepset-prompt-injections", "model_name": "piguard", "accuracy": 0.8517110266159695, "accuracy_per_category": { "1": 0.8517110266159695 }, "fpr": 0.0, "mean_latency": 0.03249339826877744, "latency_ci_95": { "lower": 0.03210889917853674, "upper": 0.03287789735901814, "mean": 0.03249339826877744, "std_dev": 0.005047406003291573, "n": 662 }, "provider": "RunPod", "model_type": "specialized", "model_developer": "Leolee99", "model_url": "https://huggingface.co/leolee99/PIGuard", "cost_info": { "cost_per_1M_input_tokens": 0.6934719574733363, "cost_per_1M_output_tokens": 0.0, "cost_per_h": 2.39, "cost_source": "RunPod", "cost_additional_info": "Output token cost is disregarded. The cost per 1M input tokens is estimated as total_cost * (1,000,000 / total_input_tokens).", "total_cost": 0.014280668020248414 }, "execution_specifications": { "type": "Local", "details": "This model was ran on an H100 (80GB PCIe) on RunPod using the transformers Library." }, "num_samples": 662 }, "leolee99/local_datasets-malignant/piguard": { "model_provider_use_case": "leolee99", "dataset_name": "local_datasets-malignant", "model_name": "piguard", "accuracy": 0.9899497487437185, "accuracy_per_category": { "jailbreak": 0.9899497487437185 }, "fpr": 0.005852231163130944, "mean_latency": 0.027675895855344575, "latency_ci_95": { "lower": 0.02739681522433326, "upper": 0.027954976486355892, "mean": 0.027675895855344575, "std_dev": 0.005634683201702699, "n": 1566 }, "provider": "RunPod", "model_type": "specialized", "model_developer": "Leolee99", "model_url": "https://huggingface.co/leolee99/PIGuard", "cost_info": { "cost_per_1M_input_tokens": 0.3051569108708133, "cost_per_1M_output_tokens": 0.0, "cost_per_h": 2.39, "cost_source": "RunPod", "cost_additional_info": "Output token cost is disregarded. The cost per 1M input tokens is estimated as total_cost * (1,000,000 / total_input_tokens).", "total_cost": 0.028773245126008987 }, "execution_specifications": { "type": "Local", "details": "This model was ran on an H100 (80GB PCIe) on RunPod using the transformers Library." }, "num_samples": 1566 }, "meta-llama/TrustAIRLab-in-the-wild-jailbreak-prompts/llama-prompt-guard-2-22m": { "model_provider_use_case": "meta-llama", "dataset_name": "TrustAIRLab-in-the-wild-jailbreak-prompts", "model_name": "llama-prompt-guard-2-22m", "accuracy": 0.8929618768328446, "accuracy_per_category": { "true": 0.8929618768328446 }, "fpr": "N/A", "mean_latency": 0.024935610133531856, "latency_ci_95": { "lower": 0.022657520362687998, "upper": 0.027213699904375714, "mean": 0.024935610133531856, "std_dev": 0.042926150884882976, "n": 1364 }, "provider": "RunPod", "model_type": "specialized", "model_developer": "Meta", "model_url": "https://huggingface.co/meta-llama/Llama-Prompt-Guard-2-22M", "cost_info": { "cost_per_1M_input_tokens": 0.026101198263857102, "cost_per_1M_output_tokens": 0.0, "cost_per_h": 2.39, "cost_source": "RunPod", "cost_additional_info": "Output token cost is disregarded. The cost per 1M input tokens is estimated as total_cost * (1,000,000 / total_input_tokens).", "total_cost": 0.02258030322525236 }, "execution_specifications": { "type": "Local", "details": "This model was ran on an H100 (80GB PCIe) on RunPod using the transformers Library." }, "num_samples": 1364 }, "meta-llama/TrustAIRLab-in-the-wild-jailbreak-prompts/llama-prompt-guard-2-86m": { "model_provider_use_case": "meta-llama", "dataset_name": "TrustAIRLab-in-the-wild-jailbreak-prompts", "model_name": "llama-prompt-guard-2-86m", "accuracy": 0.9310850439882697, "accuracy_per_category": { "true": 0.9310850439882697 }, "fpr": "N/A", "mean_latency": 0.034819692635466276, "latency_ci_95": { "lower": 0.029364393487528277, "upper": 0.040274991783404275, "mean": 0.034819692635466276, "std_dev": 0.10279445408326296, "n": 1364 }, "provider": "RunPod", "model_type": "specialized", "model_developer": "Meta", "model_url": "https://huggingface.co/meta-llama/Llama-Prompt-Guard-2-86M", "cost_info": { "cost_per_1M_input_tokens": 0.03699918472769322, "cost_per_1M_output_tokens": 0.0, "cost_per_h": 2.39, "cost_source": "RunPod", "cost_additional_info": "Output token cost is disregarded. The cost per 1M input tokens is estimated as total_cost * (1,000,000 / total_input_tokens).", "total_cost": 0.03153077922330962 }, "execution_specifications": { "type": "Local", "details": "This model was ran on an H100 (80GB PCIe) on RunPod using the transformers Library." }, "num_samples": 1364 }, "meta-llama/allenai-wildjailbreak/llama-prompt-guard-2-22m": { "model_provider_use_case": "meta-llama", "dataset_name": "allenai-wildjailbreak", "model_name": "llama-prompt-guard-2-22m", "accuracy": 0.2675, "accuracy_per_category": { "adversarial_harmful": 0.2675 }, "fpr": 0.16666666666666666, "mean_latency": 0.029936639755559723, "latency_ci_95": { "lower": 0.02971407106503298, "upper": 0.030159208446086466, "mean": 0.029936639755559723, "std_dev": 0.005338314254529593, "n": 2210 }, "provider": "RunPod", "model_type": "specialized", "model_developer": "Meta", "model_url": "https://huggingface.co/meta-llama/Llama-Prompt-Guard-2-22M", "cost_info": { "cost_per_1M_input_tokens": 0.12105300279652718, "cost_per_1M_output_tokens": 0.0, "cost_per_h": 2.39, "cost_source": "RunPod", "cost_additional_info": "Output token cost is disregarded. The cost per 1M input tokens is estimated as total_cost * (1,000,000 / total_input_tokens).", "total_cost": 0.043922871534691925 }, "execution_specifications": { "type": "Local", "details": "This model was ran on an H100 (80GB PCIe) on RunPod using the transformers Library." }, "num_samples": 2210 }, "meta-llama/allenai-wildjailbreak/llama-prompt-guard-2-86m": { "model_provider_use_case": "meta-llama", "dataset_name": "allenai-wildjailbreak", "model_name": "llama-prompt-guard-2-86m", "accuracy": 0.4375, "accuracy_per_category": { "adversarial_harmful": 0.4375 }, "fpr": 0.2857142857142857, "mean_latency": 0.030707455868095295, "latency_ci_95": { "lower": 0.03049881535858811, "upper": 0.03091609637760248, "mean": 0.030707455868095295, "std_dev": 0.005004246569176329, "n": 2210 }, "provider": "RunPod", "model_type": "specialized", "model_developer": "Meta", "model_url": "https://huggingface.co/meta-llama/Llama-Prompt-Guard-2-86M", "cost_info": { "cost_per_1M_input_tokens": 0.12152236349159765, "cost_per_1M_output_tokens": 0.0, "cost_per_h": 2.39, "cost_source": "RunPod", "cost_additional_info": "Output token cost is disregarded. The cost per 1M input tokens is estimated as total_cost * (1,000,000 / total_input_tokens).", "total_cost": 0.04505380865269237 }, "execution_specifications": { "type": "Local", "details": "This model was ran on an H100 (80GB PCIe) on RunPod using the transformers Library." }, "num_samples": 2210 }, "meta-llama/centrepourlasecuriteia-BET-jailbreak-dataset/llama-prompt-guard-2-22m": { "model_provider_use_case": "meta-llama", "dataset_name": "centrepourlasecuriteia-BET-jailbreak-dataset", "model_name": "llama-prompt-guard-2-22m", "accuracy": 0.44406656139924694, "accuracy_per_category": { "privacy": 0.3239263803680982, "harm to minors": 0.38414634146341464, "integrity & quality violations": 0.850546780072904, "illegal activities": 0.45054945054945056, "sexual content": 0.40240963855421685, "physical harm": 0.37604790419161677, "cbrn": 0.39927184466019416, "harmful manipulation": 0.4969987995198079, "cyber": 0.3916256157635468, "self-harm": 0.3637469586374696 }, "fpr": 0.5047846889952153, "mean_latency": 0.030342843858813723, "latency_ci_95": { "lower": 0.030115398571848133, "upper": 0.030570289145779313, "mean": 0.030342843858813723, "std_dev": 0.011050974367290842, "n": 9069 }, "provider": "RunPod", "model_type": "specialized", "model_developer": "Meta", "model_url": "https://huggingface.co/meta-llama/Llama-Prompt-Guard-2-22M", "cost_info": { "cost_per_1M_input_tokens": 0.06105240250286494, "cost_per_1M_output_tokens": 0.0, "cost_per_h": 2.39, "cost_source": "RunPod", "cost_additional_info": "Output token cost is disregarded. The cost per 1M input tokens is estimated as total_cost * (1,000,000 / total_input_tokens).", "total_cost": 0.18268844716217783 }, "execution_specifications": { "type": "Local", "details": "This model was ran on an H100 (80GB PCIe) on RunPod using the transformers Library." }, "num_samples": 9069 }, "meta-llama/centrepourlasecuriteia-BET-jailbreak-dataset/llama-prompt-guard-2-86m": { "model_provider_use_case": "meta-llama", "dataset_name": "centrepourlasecuriteia-BET-jailbreak-dataset", "model_name": "llama-prompt-guard-2-86m", "accuracy": 0.543301348232722, "accuracy_per_category": { "privacy": 0.4343558282208589, "harm to minors": 0.4609756097560976, "integrity & quality violations": 0.8748481166464156, "illegal activities": 0.6532356532356532, "sexual content": 0.4542168674698795, "physical harm": 0.444311377245509, "cbrn": 0.4696601941747573, "harmful manipulation": 0.673469387755102, "cyber": 0.5615763546798029, "self-harm": 0.40632603406326034 }, "fpr": 0.5155502392344498, "mean_latency": 0.033881847670985524, "latency_ci_95": { "lower": 0.03319460499468161, "upper": 0.03456909034728944, "mean": 0.033881847670985524, "std_dev": 0.03339133248820331, "n": 9069 }, "provider": "RunPod", "model_type": "specialized", "model_developer": "Meta", "model_url": "https://huggingface.co/meta-llama/Llama-Prompt-Guard-2-86M", "cost_info": { "cost_per_1M_input_tokens": 0.07426828356396996, "cost_per_1M_output_tokens": 0.0, "cost_per_h": 2.39, "cost_source": "RunPod", "cost_additional_info": "Output token cost is disregarded. The cost per 1M input tokens is estimated as total_cost * (1,000,000 / total_input_tokens).", "total_cost": 0.20399611080620023 }, "execution_specifications": { "type": "Local", "details": "This model was ran on an H100 (80GB PCIe) on RunPod using the transformers Library." }, "num_samples": 9069 }, "meta-llama/centrepourlasecuriteia-jailbreak-dataset/llama-prompt-guard-2-22m": { "model_provider_use_case": "meta-llama", "dataset_name": "centrepourlasecuriteia-jailbreak-dataset", "model_name": "llama-prompt-guard-2-22m", "accuracy": 0.14309061194842926, "accuracy_per_category": { "integrity & quality violations": 0.16853932584269662, "physical harm": 0.10338345864661654, "harm to minors": 0.15849673202614378, "illegal activities": 0.1417910447761194, "cyber": 0.18045112781954886, "privacy": 0.15327102803738318, "sexual content": 0.13271028037383178, "cbrn": 0.13992537313432835, "self-harm": 0.10734463276836158, "harmful manipulation": 0.1383177570093458 }, "fpr": 0.11790878754171301, "mean_latency": 0.03165891049170993, "latency_ci_95": { "lower": 0.031033870661128192, "upper": 0.032283950322291664, "mean": 0.03165891049170993, "std_dev": 0.025523785688390324, "n": 6406 }, "provider": "RunPod", "model_type": "specialized", "model_developer": "Meta", "model_url": "https://huggingface.co/meta-llama/Llama-Prompt-Guard-2-22M", "cost_info": { "cost_per_1M_input_tokens": 0.04970505481790414, "cost_per_1M_output_tokens": 0.0, "cost_per_h": 2.39, "cost_source": "RunPod", "cost_additional_info": "Output token cost is disregarded. The cost per 1M input tokens is estimated as total_cost * (1,000,000 / total_input_tokens).", "total_cost": 0.13464130101601282 }, "execution_specifications": { "type": "Local", "details": "This model was ran on an H100 (80GB PCIe) on RunPod using the transformers Library." }, "num_samples": 6406, "accuracy_per_technique": { "ascii_art": 0.0, "fsh": 0.025806451612903226, "adversarial_suffixes": 0.1, "tokenbreak": 0.024193548387096774, "structural_obfuscation": 0.38225806451612904, "cognitive_psychological": 0.7048387096774194, "low_resource_language": 0.0018281535648994515, "encoding_cyphering": 0.0, "dap": 0.03225806451612903 } }, "meta-llama/centrepourlasecuriteia-jailbreak-dataset/llama-prompt-guard-2-86m": { "model_provider_use_case": "meta-llama", "dataset_name": "centrepourlasecuriteia-jailbreak-dataset", "model_name": "llama-prompt-guard-2-86m", "accuracy": 0.1765026330125295, "accuracy_per_category": { "integrity & quality violations": 0.23274478330658105, "physical harm": 0.10714285714285714, "harm to minors": 0.21568627450980393, "illegal activities": 0.166044776119403, "cyber": 0.2161654135338346, "privacy": 0.1925233644859813, "sexual content": 0.1588785046728972, "cbrn": 0.14365671641791045, "self-harm": 0.12994350282485875, "harmful manipulation": 0.18691588785046728 }, "fpr": 0.12903225806451613, "mean_latency": 0.030342599208973513, "latency_ci_95": { "lower": 0.029700581120224515, "upper": 0.030984617297722512, "mean": 0.030342599208973513, "std_dev": 0.026217100580691047, "n": 6406 }, "provider": "RunPod", "model_type": "specialized", "model_developer": "Meta", "model_url": "https://huggingface.co/meta-llama/Llama-Prompt-Guard-2-86M", "cost_info": { "cost_per_1M_input_tokens": 0.06420982784383697, "cost_per_1M_output_tokens": 0.0, "cost_per_h": 2.39, "cost_source": "RunPod", "cost_additional_info": "Output token cost is disregarded. The cost per 1M input tokens is estimated as total_cost * (1,000,000 / total_input_tokens).", "total_cost": 0.12904319732586544 }, "execution_specifications": { "type": "Local", "details": "This model was ran on an H100 (80GB PCIe) on RunPod using the transformers Library." }, "num_samples": 6406, "accuracy_per_technique": { "ascii_art": 0.0, "fsh": 0.024193548387096774, "adversarial_suffixes": 0.1032258064516129, "tokenbreak": 0.18548387096774194, "structural_obfuscation": 0.5290322580645161, "cognitive_psychological": 0.6887096774193548, "low_resource_language": 0.009140767824497258, "encoding_cyphering": 0.0, "dap": 0.02903225806451613 } }, "meta-llama/deepset-prompt-injections/llama-prompt-guard-2-22m": { "model_provider_use_case": "meta-llama", "dataset_name": "deepset-prompt-injections", "model_name": "llama-prompt-guard-2-22m", "accuracy": 0.19391634980988592, "accuracy_per_category": { "1": 0.19391634980988592 }, "fpr": 0.002506265664160401, "mean_latency": 0.023843962980541217, "latency_ci_95": { "lower": 0.0235036225934286, "upper": 0.024184303367653835, "mean": 0.023843962980541217, "std_dev": 0.004467724779269086, "n": 662 }, "provider": "RunPod", "model_type": "specialized", "model_developer": "Meta", "model_url": "https://huggingface.co/meta-llama/Llama-Prompt-Guard-2-22M", "cost_info": { "cost_per_1M_input_tokens": 0.4755099947130803, "cost_per_1M_output_tokens": 0.0, "cost_per_h": 2.39, "cost_source": "RunPod", "cost_additional_info": "Output token cost is disregarded. The cost per 1M input tokens is estimated as total_cost * (1,000,000 / total_input_tokens).", "total_cost": 0.010479289263486864 }, "execution_specifications": { "type": "Local", "details": "This model was ran on an H100 (80GB PCIe) on RunPod using the transformers Library." }, "num_samples": 662 }, "meta-llama/deepset-prompt-injections/llama-prompt-guard-2-86m": { "model_provider_use_case": "meta-llama", "dataset_name": "deepset-prompt-injections", "model_name": "llama-prompt-guard-2-86m", "accuracy": 0.22813688212927757, "accuracy_per_category": { "1": 0.22813688212927757 }, "fpr": 0.002506265664160401, "mean_latency": 0.03157642780834094, "latency_ci_95": { "lower": 0.03120091028046878, "upper": 0.0319519453362131, "mean": 0.03157642780834094, "std_dev": 0.004929503014783769, "n": 662 }, "provider": "RunPod", "model_type": "specialized", "model_developer": "Meta", "model_url": "https://huggingface.co/meta-llama/Llama-Prompt-Guard-2-86M", "cost_info": { "cost_per_1M_input_tokens": 0.697299999857648, "cost_per_1M_output_tokens": 0.0, "cost_per_h": 2.39, "cost_source": "RunPod", "cost_additional_info": "Output token cost is disregarded. The cost per 1M input tokens is estimated as total_cost * (1,000,000 / total_input_tokens).", "total_cost": 0.013877664597166911 }, "execution_specifications": { "type": "Local", "details": "This model was ran on an H100 (80GB PCIe) on RunPod using the transformers Library." }, "num_samples": 662 }, "meta-llama/local_datasets-malignant/llama-prompt-guard-2-22m": { "model_provider_use_case": "meta-llama", "dataset_name": "local_datasets-malignant", "model_name": "llama-prompt-guard-2-22m", "accuracy": 0.914572864321608, "accuracy_per_category": { "jailbreak": 0.914572864321608 }, "fpr": 0.0043891733723482075, "mean_latency": 0.022609848781259274, "latency_ci_95": { "lower": 0.02241783058111005, "upper": 0.0228018669814085, "mean": 0.022609848781259274, "std_dev": 0.0038768786027223497, "n": 1566 }, "provider": "RunPod", "model_type": "specialized", "model_developer": "Meta", "model_url": "https://huggingface.co/meta-llama/Llama-Prompt-Guard-2-22M", "cost_info": { "cost_per_1M_input_tokens": 0.22907303304035678, "cost_per_1M_output_tokens": 0.0, "cost_per_h": 2.39, "cost_source": "RunPod", "cost_additional_info": "Output token cost is disregarded. The cost per 1M input tokens is estimated as total_cost * (1,000,000 / total_input_tokens).", "total_cost": 0.02350632928543621 }, "execution_specifications": { "type": "Local", "details": "This model was ran on an H100 (80GB PCIe) on RunPod using the transformers Library." }, "num_samples": 1566 }, "meta-llama/local_datasets-malignant/llama-prompt-guard-2-86m": { "model_provider_use_case": "meta-llama", "dataset_name": "local_datasets-malignant", "model_name": "llama-prompt-guard-2-86m", "accuracy": 0.9346733668341709, "accuracy_per_category": { "jailbreak": 0.9346733668341709 }, "fpr": 0.0021945866861741038, "mean_latency": 0.02865501792951562, "latency_ci_95": { "lower": 0.028355887975334733, "upper": 0.028954147883696504, "mean": 0.02865501792951562, "std_dev": 0.00603948228811614, "n": 1566 }, "provider": "RunPod", "model_type": "specialized", "model_developer": "Meta", "model_url": "https://huggingface.co/meta-llama/Llama-Prompt-Guard-2-86M", "cost_info": { "cost_per_1M_input_tokens": 0.2846255721941846, "cost_per_1M_output_tokens": 0.0, "cost_per_h": 2.39, "cost_source": "RunPod", "cost_additional_info": "Output token cost is disregarded. The cost per 1M input tokens is estimated as total_cost * (1,000,000 / total_input_tokens).", "total_cost": 0.029791189390420914 }, "execution_specifications": { "type": "Local", "details": "This model was ran on an H100 (80GB PCIe) on RunPod using the transformers Library." }, "num_samples": 1566 }, "mistral/TrustAIRLab-in-the-wild-jailbreak-prompts/ministral-14b-2512": { "model_provider_use_case": "mistral", "dataset_name": "TrustAIRLab-in-the-wild-jailbreak-prompts", "model_name": "ministral-14b-2512", "accuracy": 0.7426686217008798, "accuracy_per_category": { "true": 0.7426686217008798 }, "fpr": "N/A", "mean_latency": 0.8425018991892639, "latency_ci_95": { "lower": 0.45635083956806544, "upper": 1.2286529588104624, "mean": 0.8425018991892639, "std_dev": 7.276262271050396, "n": 1364 }, "provider": "Mistral", "model_type": "generalist", "model_developer": "Mistral AI", "model_url": "https://docs.mistral.ai/models/ministral-3-14b-25-12", "cost_info": { "cost_per_1M_input_tokens": 0.2, "cost_per_1M_output_tokens": 0.2, "cost_per_h": "N/A", "cost_source": "Ministral API pricing", "cost_additional_info": "", "total_cost": 0.18003460000000002 }, "execution_specifications": { "type": "API", "details": "REST API accessed through a CPU3 RunPod instance at US-KS-2." }, "num_samples": 1364 }, "mistral/TrustAIRLab-in-the-wild-jailbreak-prompts/ministral-3b-2512": { "model_provider_use_case": "mistral", "dataset_name": "TrustAIRLab-in-the-wild-jailbreak-prompts", "model_name": "ministral-3b-2512", "accuracy": 0.7353372434017595, "accuracy_per_category": { "true": 0.7353372434017595 }, "fpr": "N/A", "mean_latency": 0.5696844334225003, "latency_ci_95": { "lower": 0.44336821306236823, "upper": 0.6960006537826324, "mean": 0.5696844334225003, "std_dev": 0.7598195278080964, "n": 139 }, "provider": "Mistral", "model_type": "generalist", "model_developer": "Mistral AI", "model_url": "https://docs.mistral.ai/models/ministral-3-3b-25-12", "cost_info": { "cost_per_1M_input_tokens": 0.1, "cost_per_1M_output_tokens": 0.1, "cost_per_h": "N/A", "cost_source": "Ministral API pricing", "cost_additional_info": "", "total_cost": 0.10082350000000001 }, "execution_specifications": { "type": "API", "details": "REST API accessed through a CPU3 RunPod instance at US-KS-2." }, "num_samples": 1364 }, "mistral/TrustAIRLab-in-the-wild-jailbreak-prompts/mistral-large-3": { "model_provider_use_case": "mistral", "dataset_name": "TrustAIRLab-in-the-wild-jailbreak-prompts", "model_name": "mistral-large-3", "accuracy": 0.7243401759530792, "accuracy_per_category": { "true": 0.7243401759530792 }, "fpr": "N/A", "mean_latency": 1.5691638044316134, "latency_ci_95": { "lower": 1.086045915613225, "upper": 2.0522816932500016, "mean": 1.5691638044316134, "std_dev": 2.9060591356443943, "n": 139 }, "provider": "Mistral", "model_type": "generalist", "model_developer": "Mistral AI", "model_url": "https://docs.mistral.ai/models/mistral-large-3-25-12", "cost_info": { "cost_per_1M_input_tokens": 0.5, "cost_per_1M_output_tokens": 1.5, "cost_per_h": "N/A", "cost_source": "Mistral API pricing", "cost_additional_info": "", "total_cost": 0.445149 }, "execution_specifications": { "type": "API", "details": "REST API accessed through a CPU3 RunPod instance at US-KS-2." }, "num_samples": 1364 }, "mistral/allenai-wildjailbreak/ministral-14b-2512": { "model_provider_use_case": "mistral", "dataset_name": "allenai-wildjailbreak", "model_name": "ministral-14b-2512", "accuracy": 0.4885, "accuracy_per_category": { "adversarial_harmful": 0.4885 }, "fpr": 0.04285714285714286, "mean_latency": 0.46561984573554127, "latency_ci_95": { "lower": 0.4496773883206079, "upper": 0.48156230315047466, "mean": 0.46561984573554127, "std_dev": 0.38238014281772154, "n": 2210 }, "provider": "Mistral", "model_type": "generalist", "model_developer": "Mistral AI", "model_url": "https://docs.mistral.ai/models/ministral-3-14b-25-12", "cost_info": { "cost_per_1M_input_tokens": 0.2, "cost_per_1M_output_tokens": 0.2, "cost_per_h": "N/A", "cost_source": "Ministral API pricing", "cost_additional_info": "", "total_cost": 0.0831592 }, "execution_specifications": { "type": "API", "details": "REST API accessed through a CPU3 RunPod instance at US-KS-2." }, "num_samples": 2210 }, "mistral/allenai-wildjailbreak/ministral-3b-2512": { "model_provider_use_case": "mistral", "dataset_name": "allenai-wildjailbreak", "model_name": "ministral-3b-2512", "accuracy": 0.619, "accuracy_per_category": { "adversarial_harmful": 0.619 }, "fpr": 0.11904761904761904, "mean_latency": 0.3671431023610663, "latency_ci_95": { "lower": 0.34052526830590046, "upper": 0.3937609364162321, "mean": 0.3671431023610663, "std_dev": 0.20188905661674014, "n": 221 }, "provider": "Mistral", "model_type": "generalist", "model_developer": "Mistral AI", "model_url": "https://docs.mistral.ai/models/ministral-3-3b-25-12", "cost_info": { "cost_per_1M_input_tokens": 0.1, "cost_per_1M_output_tokens": 0.1, "cost_per_h": "N/A", "cost_source": "Ministral API pricing", "cost_additional_info": "", "total_cost": 0.0415798 }, "execution_specifications": { "type": "API", "details": "REST API accessed through a CPU3 RunPod instance at US-KS-2." }, "num_samples": 2210 }, "mistral/allenai-wildjailbreak/mistral-large-3": { "model_provider_use_case": "mistral", "dataset_name": "allenai-wildjailbreak", "model_name": "mistral-large-3", "accuracy": 0.527, "accuracy_per_category": { "adversarial_harmful": 0.527 }, "fpr": 0.014285714285714285, "mean_latency": 0.8031148004316097, "latency_ci_95": { "lower": 0.7156538620757189, "upper": 0.8905757387875004, "mean": 0.8031148004316097, "std_dev": 0.6633675113794177, "n": 221 }, "provider": "Mistral", "model_type": "generalist", "model_developer": "Mistral AI", "model_url": "https://docs.mistral.ai/models/mistral-large-3-25-12", "cost_info": { "cost_per_1M_input_tokens": 0.5, "cost_per_1M_output_tokens": 1.5, "cost_per_h": "N/A", "cost_source": "Mistral API pricing", "cost_additional_info": "", "total_cost": 0.212318 }, "execution_specifications": { "type": "API", "details": "REST API accessed through a CPU3 RunPod instance at US-KS-2." }, "num_samples": 2210 }, "mistral/centrepourlasecuriteia-BET-jailbreak-dataset/ministral-14b-2512": { "model_provider_use_case": "mistral", "dataset_name": "centrepourlasecuriteia-BET-jailbreak-dataset", "model_name": "ministral-14b-2512", "accuracy": 0.8327462650309729, "accuracy_per_category": { "privacy": 0.7177914110429447, "harm to minors": 0.8390243902439024, "integrity & quality violations": 0.8298906439854192, "illegal activities": 0.905982905982906, "sexual content": 0.9108433734939759, "physical harm": 0.8311377245508982, "cbrn": 0.8531553398058253, "harmful manipulation": 0.9027611044417767, "cyber": 0.6761083743842364, "self-harm": 0.8564476885644768 }, "fpr": 0.11722488038277512, "mean_latency": 0.4486026803350117, "latency_ci_95": { "lower": 0.4402562461994314, "upper": 0.45694911447059205, "mean": 0.4486026803350117, "std_dev": 0.40553150571342955, "n": 9069 }, "provider": "Mistral", "model_type": "generalist", "model_developer": "Mistral AI", "model_url": "https://docs.mistral.ai/models/ministral-3-14b-25-12", "cost_info": { "cost_per_1M_input_tokens": 0.2, "cost_per_1M_output_tokens": 0.2, "cost_per_h": "N/A", "cost_source": "Ministral API pricing", "cost_additional_info": "", "total_cost": 0.771522 }, "execution_specifications": { "type": "API", "details": "REST API accessed through a CPU3 RunPod instance at US-KS-2." }, "num_samples": 9069 }, "mistral/centrepourlasecuriteia-BET-jailbreak-dataset/ministral-3b-2512": { "model_provider_use_case": "mistral", "dataset_name": "centrepourlasecuriteia-BET-jailbreak-dataset", "model_name": "ministral-3b-2512", "accuracy": 0.8492651524353213, "accuracy_per_category": { "privacy": 0.6638036809815951, "harm to minors": 0.8792682926829268, "integrity & quality violations": 0.8955042527339003, "illegal activities": 0.9133089133089133, "sexual content": 0.9060240963855422, "physical harm": 0.8610778443113772, "cbrn": 0.9089805825242718, "harmful manipulation": 0.9255702280912365, "cyber": 0.7130541871921182, "self-harm": 0.8211678832116789 }, "fpr": 0.3576555023923445, "mean_latency": 0.44998118419520905, "latency_ci_95": { "lower": 0.42491574838049156, "upper": 0.47504662000992653, "mean": 0.44998118419520905, "std_dev": 0.3849313542438015, "n": 906 }, "provider": "Mistral", "model_type": "generalist", "model_developer": "Mistral AI", "model_url": "https://docs.mistral.ai/models/ministral-3-3b-25-12", "cost_info": { "cost_per_1M_input_tokens": 0.1, "cost_per_1M_output_tokens": 0.1, "cost_per_h": "N/A", "cost_source": "Ministral API pricing", "cost_additional_info": "", "total_cost": 0.39003960000000004 }, "execution_specifications": { "type": "API", "details": "REST API accessed through a CPU3 RunPod instance at US-KS-2." }, "num_samples": 9069 }, "mistral/centrepourlasecuriteia-BET-jailbreak-dataset/mistral-large-3": { "model_provider_use_case": "mistral", "dataset_name": "centrepourlasecuriteia-BET-jailbreak-dataset", "model_name": "mistral-large-3", "accuracy": 0.8207214866998664, "accuracy_per_category": { "privacy": 0.6417177914110429, "harm to minors": 0.7207317073170731, "integrity & quality violations": 0.8554070473876063, "illegal activities": 0.9084249084249084, "sexual content": 0.9204819277108434, "physical harm": 0.829940119760479, "cbrn": 0.8470873786407767, "harmful manipulation": 0.9015606242496998, "cyber": 0.75, "self-harm": 0.8272506082725061 }, "fpr": 0.09330143540669857, "mean_latency": 0.7457362784455154, "latency_ci_95": { "lower": 0.6806260604605588, "upper": 0.8108464964304719, "mean": 0.7457362784455154, "std_dev": 0.9999014008502649, "n": 906 }, "provider": "Mistral", "model_type": "generalist", "model_developer": "Mistral AI", "model_url": "https://docs.mistral.ai/models/mistral-large-3-25-12", "cost_info": { "cost_per_1M_input_tokens": 0.5, "cost_per_1M_output_tokens": 1.5, "cost_per_h": "N/A", "cost_source": "Mistral API pricing", "cost_additional_info": "", "total_cost": 1.9471340000000001 }, "execution_specifications": { "type": "API", "details": "REST API accessed through a CPU3 RunPod instance at US-KS-2." }, "num_samples": 9069 }, "mistral/centrepourlasecuriteia-jailbreak-dataset/ministral-14b-2512": { "model_provider_use_case": "mistral", "dataset_name": "centrepourlasecuriteia-jailbreak-dataset", "model_name": "ministral-14b-2512", "accuracy": 0.6651534410749954, "accuracy_per_category": { "integrity & quality violations": 0.579454253611557, "physical harm": 0.6804511278195489, "harm to minors": 0.7238562091503268, "illegal activities": 0.6623134328358209, "cyber": 0.6860902255639098, "privacy": 0.6261682242990654, "sexual content": 0.7065420560747664, "cbrn": 0.7089552238805971, "self-harm": 0.6440677966101694, "harmful manipulation": 0.6392523364485981 }, "fpr": 0.13125695216907676, "mean_latency": 0.48692029485693583, "latency_ci_95": { "lower": 0.46646203564166105, "upper": 0.5073785540722107, "mean": 0.48692029485693583, "std_dev": 0.8354223174580979, "n": 6406 }, "provider": "Mistral", "model_type": "generalist", "model_developer": "Mistral AI", "model_url": "https://docs.mistral.ai/models/ministral-3-14b-25-12", "cost_info": { "cost_per_1M_input_tokens": 0.2, "cost_per_1M_output_tokens": 0.2, "cost_per_h": "N/A", "cost_source": "Ministral API pricing", "cost_additional_info": "", "total_cost": 0.5702508 }, "execution_specifications": { "type": "API", "details": "REST API accessed through a CPU3 RunPod instance at US-KS-2." }, "num_samples": 6406, "accuracy_per_technique": { "ascii_art": 0.004838709677419355, "fsh": 0.9629032258064516, "adversarial_suffixes": 0.8193548387096774, "tokenbreak": 0.9064516129032258, "structural_obfuscation": 0.967741935483871, "cognitive_psychological": 0.6983870967741935, "low_resource_language": 0.3912248628884826, "encoding_cyphering": 0.3016129032258065, "dap": 0.9016129032258065 } }, "mistral/centrepourlasecuriteia-jailbreak-dataset/ministral-3b-2512": { "model_provider_use_case": "mistral", "dataset_name": "centrepourlasecuriteia-jailbreak-dataset", "model_name": "ministral-3b-2512", "accuracy": 0.7475939713092428, "accuracy_per_category": { "integrity & quality violations": 0.7207062600321027, "physical harm": 0.768796992481203, "harm to minors": 0.7679738562091504, "illegal activities": 0.75, "cyber": 0.7518796992481203, "privacy": 0.7214953271028037, "sexual content": 0.7775700934579439, "cbrn": 0.7761194029850746, "self-harm": 0.7231638418079096, "harmful manipulation": 0.719626168224299 }, "fpr": 0.407119021134594, "mean_latency": 0.4648829326033592, "latency_ci_95": { "lower": 0.4266952656241857, "upper": 0.5030705995825328, "mean": 0.4648829326033592, "std_dev": 0.4928979844171025, "n": 640 }, "provider": "Mistral", "model_type": "generalist", "model_developer": "Mistral AI", "model_url": "https://docs.mistral.ai/models/ministral-3-3b-25-12", "cost_info": { "cost_per_1M_input_tokens": 0.1, "cost_per_1M_output_tokens": 0.1, "cost_per_h": "N/A", "cost_source": "Ministral API pricing", "cost_additional_info": "", "total_cost": 0.28523489999999996 }, "execution_specifications": { "type": "API", "details": "REST API accessed through a CPU3 RunPod instance at US-KS-2." }, "num_samples": 6406, "accuracy_per_technique": { "ascii_art": 0.14193548387096774, "fsh": 0.7387096774193549, "adversarial_suffixes": 0.9532258064516129, "tokenbreak": 0.9709677419354839, "structural_obfuscation": 0.9967741935483871, "cognitive_psychological": 0.9225806451612903, "low_resource_language": 0.7385740402193784, "encoding_cyphering": 0.7774193548387097, "dap": 0.4870967741935484 } }, "mistral/centrepourlasecuriteia-jailbreak-dataset/mistral-large-3": { "model_provider_use_case": "mistral", "dataset_name": "centrepourlasecuriteia-jailbreak-dataset", "model_name": "mistral-large-3", "accuracy": 0.6540766297439622, "accuracy_per_category": { "integrity & quality violations": 0.5120385232744783, "physical harm": 0.693609022556391, "harm to minors": 0.7777777777777778, "illegal activities": 0.6623134328358209, "cyber": 0.6879699248120301, "privacy": 0.5981308411214953, "sexual content": 0.7383177570093458, "cbrn": 0.6996268656716418, "self-harm": 0.5668549905838042, "harmful manipulation": 0.6093457943925233 }, "fpr": 0.07563959955506118, "mean_latency": 0.9014793440699578, "latency_ci_95": { "lower": 0.7750448626702497, "upper": 1.0279138254696658, "mean": 0.9014793440699578, "std_dev": 1.631922187776618, "n": 640 }, "provider": "Mistral", "model_type": "generalist", "model_developer": "Mistral AI", "model_url": "https://docs.mistral.ai/models/mistral-large-3-25-12", "cost_info": { "cost_per_1M_input_tokens": 0.5, "cost_per_1M_output_tokens": 1.5, "cost_per_h": "N/A", "cost_source": "Mistral API pricing", "cost_additional_info": "", "total_cost": 1.4387694999999998 }, "execution_specifications": { "type": "API", "details": "REST API accessed through a CPU3 RunPod instance at US-KS-2." }, "num_samples": 6406, "accuracy_per_technique": { "ascii_art": 0.056451612903225805, "fsh": 0.882258064516129, "adversarial_suffixes": 0.9016129032258065, "tokenbreak": 0.9290322580645162, "structural_obfuscation": 0.9725806451612903, "cognitive_psychological": 0.6306451612903226, "low_resource_language": 0.6636197440585009, "encoding_cyphering": 0.5274193548387097, "dap": 0.3241935483870968 } }, "mistral/deepset-prompt-injections/ministral-14b-2512": { "model_provider_use_case": "mistral", "dataset_name": "deepset-prompt-injections", "model_name": "ministral-14b-2512", "accuracy": 0.22053231939163498, "accuracy_per_category": { "1": 0.22053231939163498 }, "fpr": 0.020050125313283207, "mean_latency": 0.4383037529683185, "latency_ci_95": { "lower": 0.4098312710863062, "upper": 0.4667762348503309, "mean": 0.4383037529683185, "std_dev": 0.37376467104229955, "n": 662 }, "provider": "Mistral", "model_type": "generalist", "model_developer": "Mistral AI", "model_url": "https://docs.mistral.ai/models/ministral-3-14b-25-12", "cost_info": { "cost_per_1M_input_tokens": 0.2, "cost_per_1M_output_tokens": 0.2, "cost_per_h": "N/A", "cost_source": "Ministral API pricing", "cost_additional_info": "", "total_cost": 0.0087448 }, "execution_specifications": { "type": "API", "details": "REST API accessed through a CPU3 RunPod instance at US-KS-2." }, "num_samples": 662 }, "mistral/deepset-prompt-injections/ministral-3b-2512": { "model_provider_use_case": "mistral", "dataset_name": "deepset-prompt-injections", "model_name": "ministral-3b-2512", "accuracy": 0.4828897338403042, "accuracy_per_category": { "1": 0.4828897338403042 }, "fpr": 0.13533834586466165, "mean_latency": 0.3605920032218651, "latency_ci_95": { "lower": 0.32997505959034507, "upper": 0.3912089468533851, "mean": 0.3605920032218651, "std_dev": 0.11478962660323327, "n": 54 }, "provider": "Mistral", "model_type": "generalist", "model_developer": "Mistral AI", "model_url": "https://docs.mistral.ai/models/ministral-3-3b-25-12", "cost_info": { "cost_per_1M_input_tokens": 0.1, "cost_per_1M_output_tokens": 0.1, "cost_per_h": "N/A", "cost_source": "Ministral API pricing", "cost_additional_info": "", "total_cost": 0.004468300000000001 }, "execution_specifications": { "type": "API", "details": "REST API accessed through a CPU3 RunPod instance at US-KS-2." }, "num_samples": 662 }, "mistral/deepset-prompt-injections/mistral-large-3": { "model_provider_use_case": "mistral", "dataset_name": "deepset-prompt-injections", "model_name": "mistral-large-3", "accuracy": 0.3155893536121673, "accuracy_per_category": { "1": 0.3155893536121673 }, "fpr": 0.02506265664160401, "mean_latency": 0.8135055833392673, "latency_ci_95": { "lower": 0.6907632592128088, "upper": 0.9362479074657257, "mean": 0.8135055833392673, "std_dev": 0.460187852989482, "n": 54 }, "provider": "Mistral", "model_type": "generalist", "model_developer": "Mistral AI", "model_url": "https://docs.mistral.ai/models/mistral-large-3-25-12", "cost_info": { "cost_per_1M_input_tokens": 0.5, "cost_per_1M_output_tokens": 1.5, "cost_per_h": "N/A", "cost_source": "Mistral API pricing", "cost_additional_info": "", "total_cost": 0.023185 }, "execution_specifications": { "type": "API", "details": "REST API accessed through a CPU3 RunPod instance at US-KS-2." }, "num_samples": 662 }, "mistral/local_datasets-malignant/ministral-14b-2512": { "model_provider_use_case": "mistral", "dataset_name": "local_datasets-malignant", "model_name": "ministral-14b-2512", "accuracy": 0.7537688442211056, "accuracy_per_category": { "jailbreak": 0.7537688442211056 }, "fpr": 0.008046817849305048, "mean_latency": 0.45551366976546753, "latency_ci_95": { "lower": 0.4318421342999866, "upper": 0.4791852052309485, "mean": 0.45551366976546753, "std_dev": 0.4779321401220635, "n": 1566 }, "provider": "Mistral", "model_type": "generalist", "model_developer": "Mistral AI", "model_url": "https://docs.mistral.ai/models/ministral-3-14b-25-12", "cost_info": { "cost_per_1M_input_tokens": 0.2, "cost_per_1M_output_tokens": 0.2, "cost_per_h": "N/A", "cost_source": "Ministral API pricing", "cost_additional_info": "", "total_cost": 0.0315266 }, "execution_specifications": { "type": "API", "details": "REST API accessed through a CPU3 RunPod instance at US-KS-2." }, "num_samples": 1566 }, "mistral/local_datasets-malignant/ministral-3b-2512": { "model_provider_use_case": "mistral", "dataset_name": "local_datasets-malignant", "model_name": "ministral-3b-2512", "accuracy": 0.7487437185929648, "accuracy_per_category": { "jailbreak": 0.7487437185929648 }, "fpr": 0.11558156547183614, "mean_latency": 0.4093558380875406, "latency_ci_95": { "lower": 0.35231195682368566, "upper": 0.46639971935139557, "mean": 0.4093558380875406, "std_dev": 0.36583187197062295, "n": 158 }, "provider": "Mistral", "model_type": "generalist", "model_developer": "Mistral AI", "model_url": "https://docs.mistral.ai/models/ministral-3-3b-25-12", "cost_info": { "cost_per_1M_input_tokens": 0.1, "cost_per_1M_output_tokens": 0.1, "cost_per_h": "N/A", "cost_source": "Ministral API pricing", "cost_additional_info": "", "total_cost": 0.0164296 }, "execution_specifications": { "type": "API", "details": "REST API accessed through a CPU3 RunPod instance at US-KS-2." }, "num_samples": 1566 }, "mistral/local_datasets-malignant/mistral-large-3": { "model_provider_use_case": "mistral", "dataset_name": "local_datasets-malignant", "model_name": "mistral-large-3", "accuracy": 0.7487437185929648, "accuracy_per_category": { "jailbreak": 0.7487437185929648 }, "fpr": 0.010972933430870519, "mean_latency": 1.044228754466093, "latency_ci_95": { "lower": 0.6733984399005941, "upper": 1.415059069031592, "mean": 1.044228754466093, "std_dev": 2.3781963140525555, "n": 158 }, "provider": "Mistral", "model_type": "generalist", "model_developer": "Mistral AI", "model_url": "https://docs.mistral.ai/models/mistral-large-3-25-12", "cost_info": { "cost_per_1M_input_tokens": 0.5, "cost_per_1M_output_tokens": 1.5, "cost_per_h": "N/A", "cost_source": "Mistral API pricing", "cost_additional_info": "", "total_cost": 0.08047299999999999 }, "execution_specifications": { "type": "API", "details": "REST API accessed through a CPU3 RunPod instance at US-KS-2." }, "num_samples": 1566 }, "openai/TrustAIRLab-in-the-wild-jailbreak-prompts/gpt-5-mini": { "model_provider_use_case": "openai", "dataset_name": "TrustAIRLab-in-the-wild-jailbreak-prompts", "model_name": "gpt-5-mini", "accuracy": 0.6847507331378299, "accuracy_per_category": { "true": 0.6847507331378299 }, "fpr": "N/A", "mean_latency": 1.0735461640218136, "latency_ci_95": { "lower": 1.051217270449711, "upper": 1.0958750575939162, "mean": 1.0735461640218136, "std_dev": 0.420744374008371, "n": 1364 }, "provider": "OpenAI", "model_type": "generalist", "model_developer": "OpenAI", "model_url": "https://platform.openai.com/docs/models/gpt-5-mini", "cost_info": { "cost_per_1M_input_tokens": 0.25, "cost_per_1M_output_tokens": 2.0, "cost_per_h": "N/A", "cost_source": "OpenAI API pricing", "cost_additional_info": "", "total_cost": 0.236707 }, "execution_specifications": { "type": "API", "details": "Reasoning effort set to low; see OpenAI API docs. REST API accessed through a CPU3 RunPod instance at US-KS-2." }, "num_samples": 1364 }, "openai/TrustAIRLab-in-the-wild-jailbreak-prompts/gpt-5-nano": { "model_provider_use_case": "openai", "dataset_name": "TrustAIRLab-in-the-wild-jailbreak-prompts", "model_name": "gpt-5-nano", "accuracy": 0.8203812316715543, "accuracy_per_category": { "true": 0.8203812316715543 }, "fpr": "N/A", "mean_latency": 1.6238400232877663, "latency_ci_95": { "lower": 1.5367675323478864, "upper": 1.7109125142276462, "mean": 1.6238400232877663, "std_dev": 0.5237599633870584, "n": 139 }, "provider": "OpenAI", "model_type": "generalist", "model_developer": "OpenAI", "model_url": "https://platform.openai.com/docs/models/gpt-5-nano", "cost_info": { "cost_per_1M_input_tokens": 0.05, "cost_per_1M_output_tokens": 0.4, "cost_per_h": "N/A", "cost_source": "OpenAI API pricing", "cost_additional_info": "", "total_cost": 0.102119 }, "execution_specifications": { "type": "API", "details": "Reasoning effort set to minimal; see OpenAI API docs. REST API accessed through a CPU3 RunPod instance at US-KS-2." }, "num_samples": 1364 }, "openai/TrustAIRLab-in-the-wild-jailbreak-prompts/gpt-5.2": { "model_provider_use_case": "openai", "dataset_name": "TrustAIRLab-in-the-wild-jailbreak-prompts", "model_name": "gpt-5.2", "accuracy": 0.842375366568915, "accuracy_per_category": { "true": 0.842375366568915 }, "fpr": "N/A", "mean_latency": 1.075304372705144, "latency_ci_95": { "lower": 1.027368813653646, "upper": 1.1232399317566417, "mean": 1.075304372705144, "std_dev": 0.2883428093390127, "n": 139 }, "provider": "OpenAI", "model_type": "generalist", "model_developer": "OpenAI", "model_url": "https://platform.openai.com/docs/models/gpt-5.2", "cost_info": { "cost_per_1M_input_tokens": 1.75, "cost_per_1M_output_tokens": 14.0, "cost_per_h": "N/A", "cost_source": "OpenAI API pricing", "cost_additional_info": "", "total_cost": 2.039471 }, "execution_specifications": { "type": "API", "details": "Reasoning effort set to low; see OpenAI API docs. REST API accessed through a CPU3 RunPod instance at US-KS-2." }, "num_samples": 1364 }, "openai/TrustAIRLab-in-the-wild-jailbreak-prompts/gpt-5.4": { "model_provider_use_case": "openai", "dataset_name": "TrustAIRLab-in-the-wild-jailbreak-prompts", "model_name": "gpt-5.4", "accuracy": 0.7360703812316716, "accuracy_per_category": { "true": 0.7360703812316716 }, "fpr": "N/A", "mean_latency": 0.9964801469855994, "latency_ci_95": { "lower": 0.958386425614542, "upper": 1.0345738683566568, "mean": 0.9964801469855994, "std_dev": 0.7178017531481993, "n": 1364 }, "provider": "OpenAI", "model_type": "generalist", "model_developer": "OpenAI", "model_url": "https://platform.openai.com/docs/models/gpt-5.4", "cost_info": { "cost_per_1M_input_tokens": 2.5, "cost_per_1M_output_tokens": 22.5, "cost_per_h": "N/A", "cost_source": "OpenAI API pricing", "cost_additional_info": "", "total_cost": 2.21703 }, "execution_specifications": { "type": "API", "details": "Reasoning effort set to low; see OpenAI API docs. REST API accessed through a CPU3 RunPod instance at US-KS-2." }, "num_samples": 1364 }, "openai/TrustAIRLab-in-the-wild-jailbreak-prompts/gpt-oss-120b": { "model_provider_use_case": "openai", "dataset_name": "TrustAIRLab-in-the-wild-jailbreak-prompts", "model_name": "gpt-oss-120b", "accuracy": 0.8350439882697948, "accuracy_per_category": { "true": 0.8350439882697948 }, "fpr": "N/A", "mean_latency": 2.6026064200367003, "latency_ci_95": { "lower": 2.2986799703574885, "upper": 2.906532869715912, "mean": 2.6026064200367003, "std_dev": 1.828183671307314, "n": 139 }, "provider": "Together AI", "model_type": "generalist", "model_developer": "OpenAI", "model_url": "https://www.together.ai/models/gpt-oss-120b", "cost_info": { "cost_per_1M_input_tokens": 0.05, "cost_per_1M_output_tokens": 0.2, "cost_per_h": "N/A", "cost_source": "Together AI API pricing", "cost_additional_info": "", "total_cost": 0.079649 }, "execution_specifications": { "type": "API", "details": "REST API accessed through a CPU3 RunPod instance at US-KS-2." }, "num_samples": 1364 }, "openai/TrustAIRLab-in-the-wild-jailbreak-prompts/gpt-oss-20b": { "model_provider_use_case": "openai", "dataset_name": "TrustAIRLab-in-the-wild-jailbreak-prompts", "model_name": "gpt-oss-20b", "accuracy": 0.41715542521994137, "accuracy_per_category": { "true": 0.41715542521994137 }, "fpr": "N/A", "mean_latency": 3.4009653767235846, "latency_ci_95": { "lower": 2.7430948916228823, "upper": 4.058835861824287, "mean": 3.4009653767235846, "std_dev": 3.957233995150996, "n": 139 }, "provider": "Together AI", "model_type": "generalist", "model_developer": "OpenAI", "model_url": "https://www.together.ai/models/gpt-oss-20b", "cost_info": { "cost_per_1M_input_tokens": 0.05, "cost_per_1M_output_tokens": 0.2, "cost_per_h": "N/A", "cost_source": "Together AI API pricing", "cost_additional_info": "", "total_cost": 0.12339240000000001 }, "execution_specifications": { "type": "API", "details": "REST API accessed through a CPU3 RunPod instance at US-KS-2." }, "num_samples": 1364 }, "openai/allenai-wildjailbreak/gpt-5-mini": { "model_provider_use_case": "openai", "dataset_name": "allenai-wildjailbreak", "model_name": "gpt-5-mini", "accuracy": 0.9664832416208105, "accuracy_per_category": { "adversarial_harmful": 0.9664832416208105 }, "fpr": 0.1761904761904762, "mean_latency": 0.9221740219052316, "latency_ci_95": { "lower": 0.8952255653694751, "upper": 0.9491224784409881, "mean": 0.9221740219052316, "std_dev": 0.6462129883574254, "n": 2209 }, "provider": "OpenAI", "model_type": "generalist", "model_developer": "OpenAI", "model_url": "https://platform.openai.com/docs/models/gpt-5-mini", "cost_info": { "cost_per_1M_input_tokens": 0.25, "cost_per_1M_output_tokens": 2.0, "cost_per_h": "N/A", "cost_source": "OpenAI API pricing", "cost_additional_info": "", "total_cost": 0.145918 }, "execution_specifications": { "type": "API", "details": "Reasoning effort set to low; see OpenAI API docs. REST API accessed through a CPU3 RunPod instance at US-KS-2." }, "num_samples": 2209 }, "openai/allenai-wildjailbreak/gpt-5-nano": { "model_provider_use_case": "openai", "dataset_name": "allenai-wildjailbreak", "model_name": "gpt-5-nano", "accuracy": 0.966, "accuracy_per_category": { "adversarial_harmful": 0.966 }, "fpr": 0.29523809523809524, "mean_latency": 1.7176537481368397, "latency_ci_95": { "lower": 1.624657050709884, "upper": 1.8106504455637953, "mean": 1.7176537481368397, "std_dev": 0.7053547434809687, "n": 221 }, "provider": "OpenAI", "model_type": "generalist", "model_developer": "OpenAI", "model_url": "https://platform.openai.com/docs/models/gpt-5-nano", "cost_info": { "cost_per_1M_input_tokens": 0.05, "cost_per_1M_output_tokens": 0.4, "cost_per_h": "N/A", "cost_source": "OpenAI API pricing", "cost_additional_info": "", "total_cost": 0.12475930000000002 }, "execution_specifications": { "type": "API", "details": "Reasoning effort set to minimal; see OpenAI API docs. REST API accessed through a CPU3 RunPod instance at US-KS-2." }, "num_samples": 2210 }, "openai/allenai-wildjailbreak/gpt-5.2": { "model_provider_use_case": "openai", "dataset_name": "allenai-wildjailbreak", "model_name": "gpt-5.2", "accuracy": 0.9865, "accuracy_per_category": { "adversarial_harmful": 0.9865 }, "fpr": 0.3238095238095238, "mean_latency": 1.097269012917221, "latency_ci_95": { "lower": 1.0575002339851305, "upper": 1.1370377918493113, "mean": 1.097269012917221, "std_dev": 0.3016354089802895, "n": 221 }, "provider": "OpenAI", "model_type": "generalist", "model_developer": "OpenAI", "model_url": "https://platform.openai.com/docs/models/gpt-5.2", "cost_info": { "cost_per_1M_input_tokens": 1.75, "cost_per_1M_output_tokens": 14.0, "cost_per_h": "N/A", "cost_source": "OpenAI API pricing", "cost_additional_info": "", "total_cost": 1.7212055000000002 }, "execution_specifications": { "type": "API", "details": "Reasoning effort set to low; see OpenAI API docs. REST API accessed through a CPU3 RunPod instance at US-KS-2." }, "num_samples": 2210 }, "openai/allenai-wildjailbreak/gpt-5.4": { "model_provider_use_case": "openai", "dataset_name": "allenai-wildjailbreak", "model_name": "gpt-5.4", "accuracy": 0.9674837418709354, "accuracy_per_category": { "adversarial_harmful": 0.9674837418709354 }, "fpr": 0.30952380952380953, "mean_latency": 0.9630833790279612, "latency_ci_95": { "lower": 0.9333057918586639, "upper": 0.9928609661972584, "mean": 0.9630833790279612, "std_dev": 0.7140543862025347, "n": 2209 }, "provider": "OpenAI", "model_type": "generalist", "model_developer": "OpenAI", "model_url": "https://platform.openai.com/docs/models/gpt-5.4", "cost_info": { "cost_per_1M_input_tokens": 2.5, "cost_per_1M_output_tokens": 22.5, "cost_per_h": "N/A", "cost_source": "OpenAI API pricing", "cost_additional_info": "", "total_cost": 1.2155924999999999 }, "execution_specifications": { "type": "API", "details": "Reasoning effort set to low; see OpenAI API docs. REST API accessed through a CPU3 RunPod instance at US-KS-2." }, "num_samples": 2209 }, "openai/allenai-wildjailbreak/gpt-oss-120b": { "model_provider_use_case": "openai", "dataset_name": "allenai-wildjailbreak", "model_name": "gpt-oss-120b", "accuracy": 0.9425, "accuracy_per_category": { "adversarial_harmful": 0.9425 }, "fpr": 0.2523809523809524, "mean_latency": 3.3483392286084896, "latency_ci_95": { "lower": 2.8994646274532867, "upper": 3.7972138297636926, "mean": 3.3483392286084896, "std_dev": 3.4045921835196014, "n": 221 }, "provider": "Together AI", "model_type": "generalist", "model_developer": "OpenAI", "model_url": "https://www.together.ai/models/gpt-oss-120b", "cost_info": { "cost_per_1M_input_tokens": 0.05, "cost_per_1M_output_tokens": 0.2, "cost_per_h": "N/A", "cost_source": "Together AI API pricing", "cost_additional_info": "", "total_cost": 0.0759197 }, "execution_specifications": { "type": "API", "details": "REST API accessed through a CPU3 RunPod instance at US-KS-2." }, "num_samples": 2210 }, "openai/allenai-wildjailbreak/gpt-oss-20b": { "model_provider_use_case": "openai", "dataset_name": "allenai-wildjailbreak", "model_name": "gpt-oss-20b", "accuracy": 0.3405, "accuracy_per_category": { "adversarial_harmful": 0.3405 }, "fpr": 0.14285714285714285, "mean_latency": 2.3551632020268505, "latency_ci_95": { "lower": 2.104673542958454, "upper": 2.6056528610952467, "mean": 2.3551632020268505, "std_dev": 1.8998961694914032, "n": 221 }, "provider": "Together AI", "model_type": "generalist", "model_developer": "OpenAI", "model_url": "https://www.together.ai/models/gpt-oss-20b", "cost_info": { "cost_per_1M_input_tokens": 0.05, "cost_per_1M_output_tokens": 0.2, "cost_per_h": "N/A", "cost_source": "Together AI API pricing", "cost_additional_info": "", "total_cost": 0.1236745 }, "execution_specifications": { "type": "API", "details": "REST API accessed through a CPU3 RunPod instance at US-KS-2." }, "num_samples": 2210 }, "openai/centrepourlasecuriteia-BET-jailbreak-dataset/gpt-5-mini": { "model_provider_use_case": "openai", "dataset_name": "centrepourlasecuriteia-BET-jailbreak-dataset", "model_name": "gpt-5-mini", "accuracy": 0.8909090909090909, "accuracy_per_category": { "privacy": 0.8450184501845018, "harm to minors": 0.6925925925925925, "integrity & quality violations": 0.9110840438489647, "illegal activities": 0.9423312883435583, "sexual content": 0.8685162846803377, "physical harm": 0.9326923076923077, "cbrn": 0.9621026894865525, "harmful manipulation": 0.927797833935018, "cyber": 0.9021065675340768, "self-harm": 0.9218559218559218 }, "fpr": 0.039568345323741004, "mean_latency": 0.9620919908079307, "latency_ci_95": { "lower": 0.9551175899374946, "upper": 0.9690663916783668, "mean": 0.9620919908079307, "std_dev": 0.3381198448560429, "n": 9029 }, "provider": "OpenAI", "model_type": "generalist", "model_developer": "OpenAI", "model_url": "https://platform.openai.com/docs/models/gpt-5-mini", "cost_info": { "cost_per_1M_input_tokens": 0.25, "cost_per_1M_output_tokens": 2.0, "cost_per_h": "N/A", "cost_source": "OpenAI API pricing", "cost_additional_info": "", "total_cost": 0.956074 }, "execution_specifications": { "type": "API", "details": "Reasoning effort set to low; see OpenAI API docs. REST API accessed through a CPU3 RunPod instance at US-KS-2." }, "num_samples": 9029 }, "openai/centrepourlasecuriteia-BET-jailbreak-dataset/gpt-5-nano": { "model_provider_use_case": "openai", "dataset_name": "centrepourlasecuriteia-BET-jailbreak-dataset", "model_name": "gpt-5-nano", "accuracy": 0.9197133487185716, "accuracy_per_category": { "privacy": 0.912883435582822, "harm to minors": 0.7024390243902439, "integrity & quality violations": 0.9635479951397327, "illegal activities": 0.9597069597069597, "sexual content": 0.9313253012048193, "physical harm": 0.9449101796407186, "cbrn": 0.9514563106796117, "harmful manipulation": 0.9543817527010804, "cyber": 0.9408866995073891, "self-harm": 0.9343065693430657 }, "fpr": 0.13157894736842105, "mean_latency": 2.2913325998693637, "latency_ci_95": { "lower": 2.212374774692191, "upper": 2.3702904250465364, "mean": 2.2913325998693637, "std_dev": 1.2125599091218888, "n": 906 }, "provider": "OpenAI", "model_type": "generalist", "model_developer": "OpenAI", "model_url": "https://platform.openai.com/docs/models/gpt-5-nano", "cost_info": { "cost_per_1M_input_tokens": 0.05, "cost_per_1M_output_tokens": 0.4, "cost_per_h": "N/A", "cost_source": "OpenAI API pricing", "cost_additional_info": "", "total_cost": 0.6098799500000001 }, "execution_specifications": { "type": "API", "details": "Reasoning effort set to minimal; see OpenAI API docs. REST API accessed through a CPU3 RunPod instance at US-KS-2." }, "num_samples": 9069 }, "openai/centrepourlasecuriteia-BET-jailbreak-dataset/gpt-5.2": { "model_provider_use_case": "openai", "dataset_name": "centrepourlasecuriteia-BET-jailbreak-dataset", "model_name": "gpt-5.2", "accuracy": 0.9484999392687963, "accuracy_per_category": { "privacy": 0.9374233128834356, "harm to minors": 0.7317073170731707, "integrity & quality violations": 0.9842041312272175, "illegal activities": 0.9804639804639804, "sexual content": 0.9855421686746988, "physical harm": 0.9844311377245509, "cbrn": 0.9720873786407767, "harmful manipulation": 0.9735894357743097, "cyber": 0.9753694581280788, "self-harm": 0.9586374695863747 }, "fpr": 0.07535885167464115, "mean_latency": 1.6770275010178421, "latency_ci_95": { "lower": 1.6295731195749354, "upper": 1.7244818824607488, "mean": 1.6770275010178421, "std_dev": 0.7287596931745526, "n": 906 }, "provider": "OpenAI", "model_type": "generalist", "model_developer": "OpenAI", "model_url": "https://platform.openai.com/docs/models/gpt-5.2", "cost_info": { "cost_per_1M_input_tokens": 1.75, "cost_per_1M_output_tokens": 14.0, "cost_per_h": "N/A", "cost_source": "OpenAI API pricing", "cost_additional_info": "", "total_cost": 9.76961825 }, "execution_specifications": { "type": "API", "details": "Reasoning effort set to low; see OpenAI API docs. REST API accessed through a CPU3 RunPod instance at US-KS-2." }, "num_samples": 9069 }, "openai/centrepourlasecuriteia-BET-jailbreak-dataset/gpt-5.4": { "model_provider_use_case": "openai", "dataset_name": "centrepourlasecuriteia-BET-jailbreak-dataset", "model_name": "gpt-5.4", "accuracy": 0.917769950200413, "accuracy_per_category": { "privacy": 0.9251533742331288, "harm to minors": 0.55, "integrity & quality violations": 0.9623329283110571, "illegal activities": 0.9731379731379731, "sexual content": 0.9493975903614458, "physical harm": 0.9832335329341317, "cbrn": 0.9781553398058253, "harmful manipulation": 0.9615846338535414, "cyber": 0.9655172413793104, "self-harm": 0.927007299270073 }, "fpr": 0.0215311004784689, "mean_latency": 0.7741527419353691, "latency_ci_95": { "lower": 0.7602650623927211, "upper": 0.7880404214780171, "mean": 0.7741527419353691, "std_dev": 0.6747661940788866, "n": 9069 }, "provider": "OpenAI", "model_type": "generalist", "model_developer": "OpenAI", "model_url": "https://platform.openai.com/docs/models/gpt-5.4", "cost_info": { "cost_per_1M_input_tokens": 2.5, "cost_per_1M_output_tokens": 22.5, "cost_per_h": "N/A", "cost_source": "OpenAI API pricing", "cost_additional_info": "", "total_cost": 8.5965075 }, "execution_specifications": { "type": "API", "details": "Reasoning effort set to low; see OpenAI API docs. REST API accessed through a CPU3 RunPod instance at US-KS-2." }, "num_samples": 9069 }, "openai/centrepourlasecuriteia-BET-jailbreak-dataset/gpt-oss-120b": { "model_provider_use_case": "openai", "dataset_name": "centrepourlasecuriteia-BET-jailbreak-dataset", "model_name": "gpt-oss-120b", "accuracy": 0.9113324426090125, "accuracy_per_category": { "privacy": 0.9042944785276074, "harm to minors": 0.6097560975609756, "integrity & quality violations": 0.9744835965978129, "illegal activities": 0.9597069597069597, "sexual content": 0.9493975903614458, "physical harm": 0.9437125748502994, "cbrn": 0.9538834951456311, "harmful manipulation": 0.9483793517406963, "cyber": 0.9433497536945813, "self-harm": 0.9245742092457421 }, "fpr": 0.11483253588516747, "mean_latency": 2.6044587673204096, "latency_ci_95": { "lower": 2.466679798822046, "upper": 2.7422377358187733, "mean": 2.6044587673204096, "std_dev": 2.115879624931503, "n": 906 }, "provider": "Together AI", "model_type": "generalist", "model_developer": "OpenAI", "model_url": "https://www.together.ai/models/gpt-oss-120b", "cost_info": { "cost_per_1M_input_tokens": 0.05, "cost_per_1M_output_tokens": 0.2, "cost_per_h": "N/A", "cost_source": "Together AI API pricing", "cost_additional_info": "", "total_cost": 0.41264635000000005 }, "execution_specifications": { "type": "API", "details": "REST API accessed through a CPU3 RunPod instance at US-KS-2." }, "num_samples": 9069 }, "openai/centrepourlasecuriteia-BET-jailbreak-dataset/gpt-oss-20b": { "model_provider_use_case": "openai", "dataset_name": "centrepourlasecuriteia-BET-jailbreak-dataset", "model_name": "gpt-oss-20b", "accuracy": 0.5803473824851209, "accuracy_per_category": { "privacy": 0.6282208588957056, "harm to minors": 0.4524390243902439, "integrity & quality violations": 0.5917375455650061, "illegal activities": 0.6227106227106227, "sexual content": 0.6397590361445783, "physical harm": 0.5437125748502994, "cbrn": 0.5606796116504854, "harmful manipulation": 0.5798319327731093, "cyber": 0.5997536945812808, "self-harm": 0.5851581508515815 }, "fpr": 0.11722488038277512, "mean_latency": 2.405680393541096, "latency_ci_95": { "lower": 2.2000266890066484, "upper": 2.6113340980755435, "mean": 2.405680393541096, "std_dev": 3.158235890126358, "n": 906 }, "provider": "Together AI", "model_type": "generalist", "model_developer": "OpenAI", "model_url": "https://www.together.ai/models/gpt-oss-20b", "cost_info": { "cost_per_1M_input_tokens": 0.05, "cost_per_1M_output_tokens": 0.2, "cost_per_h": "N/A", "cost_source": "Together AI API pricing", "cost_additional_info": "", "total_cost": 0.67396855 }, "execution_specifications": { "type": "API", "details": "REST API accessed through a CPU3 RunPod instance at US-KS-2." }, "num_samples": 9069 }, "openai/centrepourlasecuriteia-jailbreak-dataset/gpt-5-mini": { "model_provider_use_case": "openai", "dataset_name": "centrepourlasecuriteia-jailbreak-dataset", "model_name": "gpt-5-mini", "accuracy": 0.7198111494461594, "accuracy_per_category": { "integrity & quality violations": 0.5313001605136437, "physical harm": 0.793233082706767, "harm to minors": 0.7843137254901961, "illegal activities": 0.7705223880597015, "cyber": 0.7669172932330827, "privacy": 0.6990654205607477, "sexual content": 0.7364485981308411, "cbrn": 0.8059701492537313, "self-harm": 0.64030131826742, "harmful manipulation": 0.6915887850467289 }, "fpr": 0.011123470522803115, "mean_latency": 0.937816921934425, "latency_ci_95": { "lower": 0.9309632018488703, "upper": 0.9446706420199797, "mean": 0.937816921934425, "std_dev": 0.2798747760908342, "n": 6406 }, "provider": "OpenAI", "model_type": "generalist", "model_developer": "OpenAI", "model_url": "https://platform.openai.com/docs/models/gpt-5-mini", "cost_info": { "cost_per_1M_input_tokens": 0.25, "cost_per_1M_output_tokens": 2.0, "cost_per_h": "N/A", "cost_source": "OpenAI API pricing", "cost_additional_info": "", "total_cost": 0.799141 }, "execution_specifications": { "type": "API", "details": "Reasoning effort set to low; see OpenAI API docs. REST API accessed through a CPU3 RunPod instance at US-KS-2." }, "num_samples": 6406, "accuracy_per_technique": { "ascii_art": 0.07903225806451612, "fsh": 0.9225806451612903, "adversarial_suffixes": 0.917741935483871, "tokenbreak": 0.8709677419354839, "structural_obfuscation": 0.9290322580645162, "cognitive_psychological": 0.9241935483870968, "low_resource_language": 0.793418647166362, "encoding_cyphering": 0.2903225806451613, "dap": 0.7596774193548387 } }, "openai/centrepourlasecuriteia-jailbreak-dataset/gpt-5-nano": { "model_provider_use_case": "openai", "dataset_name": "centrepourlasecuriteia-jailbreak-dataset", "model_name": "gpt-5-nano", "accuracy": 0.8135100780824406, "accuracy_per_category": { "integrity & quality violations": 0.6436597110754414, "physical harm": 0.8552631578947368, "harm to minors": 0.8382352941176471, "illegal activities": 0.8731343283582089, "cyber": 0.8627819548872181, "privacy": 0.8186915887850468, "sexual content": 0.7981308411214953, "cbrn": 0.8731343283582089, "self-harm": 0.7815442561205274, "harmful manipulation": 0.8149532710280374 }, "fpr": 0.10456062291434928, "mean_latency": 2.3395034074783325, "latency_ci_95": { "lower": 2.2434950071795523, "upper": 2.4355118077771127, "mean": 2.3395034074783325, "std_dev": 1.2392049773605518, "n": 640 }, "provider": "OpenAI", "model_type": "generalist", "model_developer": "OpenAI", "model_url": "https://platform.openai.com/docs/models/gpt-5-nano", "cost_info": { "cost_per_1M_input_tokens": 0.05, "cost_per_1M_output_tokens": 0.4, "cost_per_h": "N/A", "cost_source": "OpenAI API pricing", "cost_additional_info": "", "total_cost": 0.421511 }, "execution_specifications": { "type": "API", "details": "Reasoning effort set to minimal; see OpenAI API docs. REST API accessed through a CPU3 RunPod instance at US-KS-2." }, "num_samples": 6406, "accuracy_per_technique": { "ascii_art": 0.08548387096774193, "fsh": 0.9306451612903226, "adversarial_suffixes": 0.9451612903225807, "tokenbreak": 0.9354838709677419, "structural_obfuscation": 0.9758064516129032, "cognitive_psychological": 0.9451612903225807, "low_resource_language": 0.8884826325411335, "encoding_cyphering": 0.7064516129032258, "dap": 0.917741935483871 } }, "openai/centrepourlasecuriteia-jailbreak-dataset/gpt-5.2": { "model_provider_use_case": "openai", "dataset_name": "centrepourlasecuriteia-jailbreak-dataset", "model_name": "gpt-5.2", "accuracy": 0.844016706010532, "accuracy_per_category": { "integrity & quality violations": 0.6725521669341894, "physical harm": 0.8909774436090225, "harm to minors": 0.8611111111111112, "illegal activities": 0.8917910447761194, "cyber": 0.881578947368421, "privacy": 0.8654205607476636, "sexual content": 0.8766355140186916, "cbrn": 0.8955223880597015, "self-harm": 0.8135593220338984, "harmful manipulation": 0.8168224299065421 }, "fpr": 0.0778642936596218, "mean_latency": 1.7436416801065207, "latency_ci_95": { "lower": 1.6863250150449207, "upper": 1.8009583451681206, "mean": 1.7436416801065207, "std_dev": 0.7398008550189835, "n": 640 }, "provider": "OpenAI", "model_type": "generalist", "model_developer": "OpenAI", "model_url": "https://platform.openai.com/docs/models/gpt-5.2", "cost_info": { "cost_per_1M_input_tokens": 1.75, "cost_per_1M_output_tokens": 14.0, "cost_per_h": "N/A", "cost_source": "OpenAI API pricing", "cost_additional_info": "", "total_cost": 7.730316999999999 }, "execution_specifications": { "type": "API", "details": "Reasoning effort set to low; see OpenAI API docs. REST API accessed through a CPU3 RunPod instance at US-KS-2." }, "num_samples": 6406, "accuracy_per_technique": { "ascii_art": 0.11612903225806452, "fsh": 0.9338709677419355, "adversarial_suffixes": 0.9451612903225807, "tokenbreak": 0.9354838709677419, "structural_obfuscation": 0.964516129032258, "cognitive_psychological": 0.95, "low_resource_language": 0.8884826325411335, "encoding_cyphering": 0.9387096774193548, "dap": 0.9290322580645162 } }, "openai/centrepourlasecuriteia-jailbreak-dataset/gpt-5.4": { "model_provider_use_case": "openai", "dataset_name": "centrepourlasecuriteia-jailbreak-dataset", "model_name": "gpt-5.4", "accuracy": 0.8342110041765026, "accuracy_per_category": { "integrity & quality violations": 0.637239165329053, "physical harm": 0.8984962406015038, "harm to minors": 0.8725490196078431, "illegal activities": 0.8992537313432836, "cyber": 0.8947368421052632, "privacy": 0.822429906542056, "sexual content": 0.8411214953271028, "cbrn": 0.9104477611940298, "self-harm": 0.7909604519774012, "harmful manipulation": 0.8018691588785046 }, "fpr": 0.04449388209121246, "mean_latency": 0.9920518395307472, "latency_ci_95": { "lower": 0.9756302221527803, "upper": 1.008473456908714, "mean": 0.9920518395307472, "std_dev": 0.6705842125642937, "n": 6406 }, "provider": "OpenAI", "model_type": "generalist", "model_developer": "OpenAI", "model_url": "https://platform.openai.com/docs/models/gpt-5.4", "cost_info": { "cost_per_1M_input_tokens": 2.5, "cost_per_1M_output_tokens": 22.5, "cost_per_h": "N/A", "cost_source": "OpenAI API pricing", "cost_additional_info": "", "total_cost": 7.297415 }, "execution_specifications": { "type": "API", "details": "Reasoning effort set to low; see OpenAI API docs. REST API accessed through a CPU3 RunPod instance at US-KS-2." }, "num_samples": 6406, "accuracy_per_technique": { "ascii_art": 0.2129032258064516, "fsh": 0.9112903225806451, "adversarial_suffixes": 0.9032258064516129, "tokenbreak": 0.9129032258064517, "structural_obfuscation": 0.9467741935483871, "cognitive_psychological": 0.9241935483870968, "low_resource_language": 0.8683729433272395, "encoding_cyphering": 0.9145161290322581, "dap": 0.917741935483871 } }, "openai/centrepourlasecuriteia-jailbreak-dataset/gpt-oss-120b": { "model_provider_use_case": "openai", "dataset_name": "centrepourlasecuriteia-jailbreak-dataset", "model_name": "gpt-oss-120b", "accuracy": 0.8320319593244961, "accuracy_per_category": { "integrity & quality violations": 0.680577849117175, "physical harm": 0.8721804511278195, "harm to minors": 0.8741830065359477, "illegal activities": 0.8563432835820896, "cyber": 0.8721804511278195, "privacy": 0.8560747663551402, "sexual content": 0.8411214953271028, "cbrn": 0.8805970149253731, "self-harm": 0.7928436911487758, "harmful manipulation": 0.8130841121495327 }, "fpr": 0.07341490545050056, "mean_latency": 3.0369778852909803, "latency_ci_95": { "lower": 2.7932966937018238, "upper": 3.280659076880137, "mean": 3.0369778852909803, "std_dev": 3.1452554627167375, "n": 640 }, "provider": "Together AI", "model_type": "generalist", "model_developer": "OpenAI", "model_url": "https://www.together.ai/models/gpt-oss-120b", "cost_info": { "cost_per_1M_input_tokens": 0.05, "cost_per_1M_output_tokens": 0.2, "cost_per_h": "N/A", "cost_source": "Together AI API pricing", "cost_additional_info": "", "total_cost": 0.3445576 }, "execution_specifications": { "type": "API", "details": "REST API accessed through a CPU3 RunPod instance at US-KS-2." }, "num_samples": 6406, "accuracy_per_technique": { "ascii_art": 0.08225806451612903, "fsh": 0.9451612903225807, "adversarial_suffixes": 0.9419354838709677, "tokenbreak": 0.9, "structural_obfuscation": 0.9661290322580646, "cognitive_psychological": 0.9354838709677419, "low_resource_language": 0.8738574040219378, "encoding_cyphering": 0.9096774193548387, "dap": 0.9387096774193548 } }, "openai/centrepourlasecuriteia-jailbreak-dataset/gpt-oss-20b": { "model_provider_use_case": "openai", "dataset_name": "centrepourlasecuriteia-jailbreak-dataset", "model_name": "gpt-oss-20b", "accuracy": 0.551298347557654, "accuracy_per_category": { "integrity & quality violations": 0.39807383627608345, "physical harm": 0.5958646616541353, "harm to minors": 0.565359477124183, "illegal activities": 0.5970149253731343, "cyber": 0.5733082706766918, "privacy": 0.5588785046728972, "sexual content": 0.5476635514018692, "cbrn": 0.6268656716417911, "self-harm": 0.5310734463276836, "harmful manipulation": 0.5420560747663551 }, "fpr": 0.07452725250278086, "mean_latency": 2.7882584508508446, "latency_ci_95": { "lower": 2.4999842840310773, "upper": 3.076532617670612, "mean": 2.7882584508508446, "std_dev": 3.720828398929809, "n": 640 }, "provider": "Together AI", "model_type": "generalist", "model_developer": "OpenAI", "model_url": "https://www.together.ai/models/gpt-oss-20b", "cost_info": { "cost_per_1M_input_tokens": 0.05, "cost_per_1M_output_tokens": 0.2, "cost_per_h": "N/A", "cost_source": "Together AI API pricing", "cost_additional_info": "", "total_cost": 0.5171226 }, "execution_specifications": { "type": "API", "details": "REST API accessed through a CPU3 RunPod instance at US-KS-2." }, "num_samples": 6406, "accuracy_per_technique": { "ascii_art": 0.04838709677419355, "fsh": 0.582258064516129, "adversarial_suffixes": 0.5967741935483871, "tokenbreak": 0.646774193548387, "structural_obfuscation": 0.7080645161290322, "cognitive_psychological": 0.4596774193548387, "low_resource_language": 0.5411334552102377, "encoding_cyphering": 0.6225806451612903, "dap": 0.7548387096774194 } }, "openai/deepset-prompt-injections/gpt-5-mini": { "model_provider_use_case": "openai", "dataset_name": "deepset-prompt-injections", "model_name": "gpt-5-mini", "accuracy": 0.23574144486692014, "accuracy_per_category": { "1": 0.23574144486692014 }, "fpr": 0.02506265664160401, "mean_latency": 0.879631361572404, "latency_ci_95": { "lower": 0.8518987296702967, "upper": 0.9073639934745112, "mean": 0.879631361572404, "std_dev": 0.36405249401797823, "n": 662 }, "provider": "OpenAI", "model_type": "generalist", "model_developer": "OpenAI", "model_url": "https://platform.openai.com/docs/models/gpt-5-mini", "cost_info": { "cost_per_1M_input_tokens": 0.25, "cost_per_1M_output_tokens": 2.0, "cost_per_h": "N/A", "cost_source": "OpenAI API pricing", "cost_additional_info": "", "total_cost": 0.024109 }, "execution_specifications": { "type": "API", "details": "Reasoning effort set to low; see OpenAI API docs. REST API accessed through a CPU3 RunPod instance at US-KS-2." }, "num_samples": 662 }, "openai/deepset-prompt-injections/gpt-5-nano": { "model_provider_use_case": "openai", "dataset_name": "deepset-prompt-injections", "model_name": "gpt-5-nano", "accuracy": 0.34980988593155893, "accuracy_per_category": { "1": 0.34980988593155893 }, "fpr": 0.03508771929824561, "mean_latency": 1.5913773201130055, "latency_ci_95": { "lower": 1.3791840737725038, "upper": 1.803570566453507, "mean": 1.5913773201130055, "std_dev": 0.7955589495901884, "n": 54 }, "provider": "OpenAI", "model_type": "generalist", "model_developer": "OpenAI", "model_url": "https://platform.openai.com/docs/models/gpt-5-nano", "cost_info": { "cost_per_1M_input_tokens": 0.05, "cost_per_1M_output_tokens": 0.4, "cost_per_h": "N/A", "cost_source": "OpenAI API pricing", "cost_additional_info": "", "total_cost": 0.028026199999999998 }, "execution_specifications": { "type": "API", "details": "Reasoning effort set to minimal; see OpenAI API docs. REST API accessed through a CPU3 RunPod instance at US-KS-2." }, "num_samples": 662 }, "openai/deepset-prompt-injections/gpt-5.2": { "model_provider_use_case": "openai", "dataset_name": "deepset-prompt-injections", "model_name": "gpt-5.2", "accuracy": 0.3650190114068441, "accuracy_per_category": { "1": 0.3650190114068441 }, "fpr": 0.03508771929824561, "mean_latency": 1.0697331428527832, "latency_ci_95": { "lower": 0.9869730334473329, "upper": 1.1524932522582336, "mean": 1.0697331428527832, "std_dev": 0.3102857741330575, "n": 54 }, "provider": "OpenAI", "model_type": "generalist", "model_developer": "OpenAI", "model_url": "https://platform.openai.com/docs/models/gpt-5.2", "cost_info": { "cost_per_1M_input_tokens": 1.75, "cost_per_1M_output_tokens": 14.0, "cost_per_h": "N/A", "cost_source": "OpenAI API pricing", "cost_additional_info": "", "total_cost": 0.35252700000000003 }, "execution_specifications": { "type": "API", "details": "Reasoning effort set to low; see OpenAI API docs. REST API accessed through a CPU3 RunPod instance at US-KS-2." }, "num_samples": 662 }, "openai/deepset-prompt-injections/gpt-5.4": { "model_provider_use_case": "openai", "dataset_name": "deepset-prompt-injections", "model_name": "gpt-5.4", "accuracy": 0.20532319391634982, "accuracy_per_category": { "1": 0.20532319391634982 }, "fpr": 0.02756892230576441, "mean_latency": 0.9136189434823673, "latency_ci_95": { "lower": 0.8661126006804054, "upper": 0.9611252862843291, "mean": 0.9136189434823673, "std_dev": 0.6236264426606091, "n": 662 }, "provider": "OpenAI", "model_type": "generalist", "model_developer": "OpenAI", "model_url": "https://platform.openai.com/docs/models/gpt-5.4", "cost_info": { "cost_per_1M_input_tokens": 2.5, "cost_per_1M_output_tokens": 22.5, "cost_per_h": "N/A", "cost_source": "OpenAI API pricing", "cost_additional_info": "", "total_cost": 0.16827 }, "execution_specifications": { "type": "API", "details": "Reasoning effort set to low; see OpenAI API docs. REST API accessed through a CPU3 RunPod instance at US-KS-2." }, "num_samples": 662 }, "openai/deepset-prompt-injections/gpt-oss-120b": { "model_provider_use_case": "openai", "dataset_name": "deepset-prompt-injections", "model_name": "gpt-oss-120b", "accuracy": 0.4448669201520912, "accuracy_per_category": { "1": 0.4448669201520912 }, "fpr": 0.03508771929824561, "mean_latency": 1.4635970106831304, "latency_ci_95": { "lower": 1.2224779552664318, "upper": 1.704716066099829, "mean": 1.4635970106831304, "std_dev": 0.9040081423971, "n": 54 }, "provider": "Together AI", "model_type": "generalist", "model_developer": "OpenAI", "model_url": "https://www.together.ai/models/gpt-oss-120b", "cost_info": { "cost_per_1M_input_tokens": 0.05, "cost_per_1M_output_tokens": 0.2, "cost_per_h": "N/A", "cost_source": "Together AI API pricing", "cost_additional_info": "", "total_cost": 0.0173386 }, "execution_specifications": { "type": "API", "details": "REST API accessed through a CPU3 RunPod instance at US-KS-2." }, "num_samples": 662 }, "openai/deepset-prompt-injections/gpt-oss-20b": { "model_provider_use_case": "openai", "dataset_name": "deepset-prompt-injections", "model_name": "gpt-oss-20b", "accuracy": 0.21292775665399238, "accuracy_per_category": { "1": 0.21292775665399238 }, "fpr": 0.040100250626566414, "mean_latency": 1.970481934370818, "latency_ci_95": { "lower": 1.4588799202291787, "upper": 2.4820839485124573, "mean": 1.970481934370818, "std_dev": 1.918107988817081, "n": 54 }, "provider": "Together AI", "model_type": "generalist", "model_developer": "OpenAI", "model_url": "https://www.together.ai/models/gpt-oss-20b", "cost_info": { "cost_per_1M_input_tokens": 0.05, "cost_per_1M_output_tokens": 0.2, "cost_per_h": "N/A", "cost_source": "Together AI API pricing", "cost_additional_info": "", "total_cost": 0.0286828 }, "execution_specifications": { "type": "API", "details": "REST API accessed through a CPU3 RunPod instance at US-KS-2." }, "num_samples": 662 }, "openai/local_datasets-malignant/gpt-5-mini": { "model_provider_use_case": "openai", "dataset_name": "local_datasets-malignant", "model_name": "gpt-5-mini", "accuracy": 0.6834170854271356, "accuracy_per_category": { "jailbreak": 0.6834170854271356 }, "fpr": 0.0073152889539136795, "mean_latency": 0.9006654570690364, "latency_ci_95": { "lower": 0.875658871416478, "upper": 0.9256720427215949, "mean": 0.9006654570690364, "std_dev": 0.5048870199189724, "n": 1566 }, "provider": "OpenAI", "model_type": "generalist", "model_developer": "OpenAI", "model_url": "https://platform.openai.com/docs/models/gpt-5-mini", "cost_info": { "cost_per_1M_input_tokens": 0.25, "cost_per_1M_output_tokens": 2.0, "cost_per_h": "N/A", "cost_source": "OpenAI API pricing", "cost_additional_info": "", "total_cost": 0.06971075 }, "execution_specifications": { "type": "API", "details": "Reasoning effort set to low; see OpenAI API docs. REST API accessed through a CPU3 RunPod instance at US-KS-2." }, "num_samples": 1566 }, "openai/local_datasets-malignant/gpt-5-nano": { "model_provider_use_case": "openai", "dataset_name": "local_datasets-malignant", "model_name": "gpt-5-nano", "accuracy": 0.9246231155778895, "accuracy_per_category": { "jailbreak": 0.9246231155778895 }, "fpr": 0.01755669348939283, "mean_latency": 1.325371988212006, "latency_ci_95": { "lower": 1.264027212319762, "upper": 1.3867167641042502, "mean": 1.325371988212006, "std_dev": 0.3934142225784692, "n": 158 }, "provider": "OpenAI", "model_type": "generalist", "model_developer": "OpenAI", "model_url": "https://platform.openai.com/docs/models/gpt-5-nano", "cost_info": { "cost_per_1M_input_tokens": 0.05, "cost_per_1M_output_tokens": 0.4, "cost_per_h": "N/A", "cost_source": "OpenAI API pricing", "cost_additional_info": "", "total_cost": 0.039107350000000006 }, "execution_specifications": { "type": "API", "details": "Reasoning effort set to minimal; see OpenAI API docs. REST API accessed through a CPU3 RunPod instance at US-KS-2." }, "num_samples": 1566 }, "openai/local_datasets-malignant/gpt-5.2": { "model_provider_use_case": "openai", "dataset_name": "local_datasets-malignant", "model_name": "gpt-5.2", "accuracy": 0.9597989949748744, "accuracy_per_category": { "jailbreak": 0.9597989949748744 }, "fpr": 0.011704462326261888, "mean_latency": 0.8135021592997298, "latency_ci_95": { "lower": 0.7865587478341285, "upper": 0.8404455707653311, "mean": 0.8135021592997298, "std_dev": 0.17279256662328926, "n": 158 }, "provider": "OpenAI", "model_type": "generalist", "model_developer": "OpenAI", "model_url": "https://platform.openai.com/docs/models/gpt-5.2", "cost_info": { "cost_per_1M_input_tokens": 1.75, "cost_per_1M_output_tokens": 14.0, "cost_per_h": "N/A", "cost_source": "OpenAI API pricing", "cost_additional_info": "", "total_cost": 0.71604925 }, "execution_specifications": { "type": "API", "details": "Reasoning effort set to low; see OpenAI API docs. REST API accessed through a CPU3 RunPod instance at US-KS-2." }, "num_samples": 1566 }, "openai/local_datasets-malignant/gpt-5.4": { "model_provider_use_case": "openai", "dataset_name": "local_datasets-malignant", "model_name": "gpt-5.4", "accuracy": 0.8592964824120602, "accuracy_per_category": { "jailbreak": 0.8592964824120602 }, "fpr": 0.010972933430870519, "mean_latency": 0.9851279414019822, "latency_ci_95": { "lower": 0.9424439079559916, "upper": 1.0278119748479728, "mean": 0.9851279414019822, "std_dev": 0.8617975578150584, "n": 1566 }, "provider": "OpenAI", "model_type": "generalist", "model_developer": "OpenAI", "model_url": "https://platform.openai.com/docs/models/gpt-5.4", "cost_info": { "cost_per_1M_input_tokens": 2.5, "cost_per_1M_output_tokens": 22.5, "cost_per_h": "N/A", "cost_source": "OpenAI API pricing", "cost_additional_info": "", "total_cost": 0.5248475 }, "execution_specifications": { "type": "API", "details": "Reasoning effort set to low; see OpenAI API docs. REST API accessed through a CPU3 RunPod instance at US-KS-2." }, "num_samples": 1566 }, "openai/local_datasets-malignant/gpt-oss-120b": { "model_provider_use_case": "openai", "dataset_name": "local_datasets-malignant", "model_name": "gpt-oss-120b", "accuracy": 0.9346733668341709, "accuracy_per_category": { "jailbreak": 0.9346733668341709 }, "fpr": 0.016825164594001463, "mean_latency": 1.68960603732097, "latency_ci_95": { "lower": 1.4813149029226007, "upper": 1.8978971717193394, "mean": 1.68960603732097, "std_dev": 1.3358055925293928, "n": 158 }, "provider": "Together AI", "model_type": "generalist", "model_developer": "OpenAI", "model_url": "https://www.together.ai/models/gpt-oss-120b", "cost_info": { "cost_per_1M_input_tokens": 0.05, "cost_per_1M_output_tokens": 0.2, "cost_per_h": "N/A", "cost_source": "Together AI API pricing", "cost_additional_info": "", "total_cost": 0.03416355 }, "execution_specifications": { "type": "API", "details": "REST API accessed through a CPU3 RunPod instance at US-KS-2." }, "num_samples": 1566 }, "openai/local_datasets-malignant/gpt-oss-20b": { "model_provider_use_case": "openai", "dataset_name": "local_datasets-malignant", "model_name": "gpt-oss-20b", "accuracy": 0.3969849246231156, "accuracy_per_category": { "jailbreak": 0.3969849246231156 }, "fpr": 0.011704462326261888, "mean_latency": 1.390318976172918, "latency_ci_95": { "lower": 1.2387310890008436, "upper": 1.5419068633449924, "mean": 1.390318976172918, "std_dev": 0.972158263139964, "n": 158 }, "provider": "Together AI", "model_type": "generalist", "model_developer": "OpenAI", "model_url": "https://www.together.ai/models/gpt-oss-20b", "cost_info": { "cost_per_1M_input_tokens": 0.05, "cost_per_1M_output_tokens": 0.2, "cost_per_h": "N/A", "cost_source": "Together AI API pricing", "cost_additional_info": "", "total_cost": 0.04666655000000001 }, "execution_specifications": { "type": "API", "details": "REST API accessed through a CPU3 RunPod instance at US-KS-2." }, "num_samples": 1566 }, "protectai/TrustAIRLab-in-the-wild-jailbreak-prompts/llm-guard": { "model_provider_use_case": "protectai", "dataset_name": "TrustAIRLab-in-the-wild-jailbreak-prompts", "model_name": "llm-guard", "accuracy": 0.7199413489736071, "accuracy_per_category": { "true": 0.7199413489736071 }, "fpr": "N/A", "mean_latency": 0.022524611516432327, "latency_ci_95": { "lower": 0.021060562309746445, "upper": 0.02398866072311821, "mean": 0.022524611516432327, "std_dev": 0.027587146895362142, "n": 1364 }, "provider": "RunPod", "model_type": "specialized", "model_developer": "ProtectAI", "model_url": "https://protectai.github.io/llm-guard/input_scanners/prompt_injection/", "cost_info": { "cost_per_1M_input_tokens": 0.022949794051342044, "cost_per_1M_output_tokens": 0.0, "cost_per_h": 2.39, "cost_source": "RunPod", "cost_additional_info": "Output token cost is disregarded. The cost per 1M input tokens is estimated as total_cost * (1,000,000 / total_input_tokens).", "total_cost": 0.020397036821974652 }, "execution_specifications": { "type": "Local", "details": "This model was ran on an H100 (80GB PCIe) on RunPod using the llm-guard Python SDK." }, "num_samples": 1364 }, "protectai/allenai-wildjailbreak/llm-guard": { "model_provider_use_case": "protectai", "dataset_name": "allenai-wildjailbreak", "model_name": "llm-guard", "accuracy": 0.5745, "accuracy_per_category": { "adversarial_harmful": 0.5745 }, "fpr": 0.30952380952380953, "mean_latency": 0.017263292079597578, "latency_ci_95": { "lower": 0.016970886764508367, "upper": 0.01755569739468679, "mean": 0.017263292079597578, "std_dev": 0.007013347016360224, "n": 2210 }, "provider": "RunPod", "model_type": "specialized", "model_developer": "ProtectAI", "model_url": "https://protectai.github.io/llm-guard/input_scanners/prompt_injection/", "cost_info": { "cost_per_1M_input_tokens": 0.06007755300838914, "cost_per_1M_output_tokens": 0.0, "cost_per_h": 2.39, "cost_source": "RunPod", "cost_additional_info": "Output token cost is disregarded. The cost per 1M input tokens is estimated as total_cost * (1,000,000 / total_input_tokens).", "total_cost": 0.025328606232007347 }, "execution_specifications": { "type": "Local", "details": "This model was ran on an H100 (80GB PCIe) on RunPod using the llm-guard Python SDK." }, "num_samples": 2210 }, "protectai/centrepourlasecuriteia-BET-jailbreak-dataset/llm-guard": { "model_provider_use_case": "protectai", "dataset_name": "centrepourlasecuriteia-BET-jailbreak-dataset", "model_name": "llm-guard", "accuracy": 0.6852909024656869, "accuracy_per_category": { "privacy": 0.7595092024539877, "harm to minors": 0.624390243902439, "integrity & quality violations": 0.8408262454434994, "illegal activities": 0.7716727716727717, "sexual content": 0.6457831325301204, "physical harm": 0.6395209580838324, "cbrn": 0.508495145631068, "harmful manipulation": 0.6158463385354142, "cyber": 0.8719211822660099, "self-harm": 0.5802919708029197 }, "fpr": 0.6375598086124402, "mean_latency": 0.018967504445609244, "latency_ci_95": { "lower": 0.01887315855246471, "upper": 0.019061850338753777, "mean": 0.018967504445609244, "std_dev": 0.004584021329741206, "n": 9069 }, "provider": "RunPod", "model_type": "specialized", "model_developer": "ProtectAI", "model_url": "https://protectai.github.io/llm-guard/input_scanners/prompt_injection/", "cost_info": { "cost_per_1M_input_tokens": 0.051051736094780864, "cost_per_1M_output_tokens": 0.0, "cost_per_h": 2.39, "cost_source": "RunPod", "cost_additional_info": "Output token cost is disregarded. The cost per 1M input tokens is estimated as total_cost * (1,000,000 / total_input_tokens).", "total_cost": 0.11419970882866118 }, "execution_specifications": { "type": "Local", "details": "This model was ran on an H100 (80GB PCIe) on RunPod using the llm-guard Python SDK." }, "num_samples": 9069 }, "protectai/centrepourlasecuriteia-jailbreak-dataset/llm-guard": { "model_provider_use_case": "protectai", "dataset_name": "centrepourlasecuriteia-jailbreak-dataset", "model_name": "llm-guard", "accuracy": 0.5572907209006719, "accuracy_per_category": { "integrity & quality violations": 0.5714285714285714, "physical harm": 0.5545112781954887, "harm to minors": 0.5424836601307189, "illegal activities": 0.542910447761194, "cyber": 0.5319548872180451, "privacy": 0.5700934579439252, "sexual content": 0.525233644859813, "cbrn": 0.5597014925373134, "self-harm": 0.6026365348399246, "harmful manipulation": 0.5719626168224299 }, "fpr": 0.5661846496106785, "mean_latency": 0.019807666233097283, "latency_ci_95": { "lower": 0.01945953881754472, "upper": 0.020155793648649844, "mean": 0.019807666233097283, "std_dev": 0.014215941308167267, "n": 6406 }, "provider": "RunPod", "model_type": "specialized", "model_developer": "ProtectAI", "model_url": "https://protectai.github.io/llm-guard/input_scanners/prompt_injection/", "cost_info": { "cost_per_1M_input_tokens": 0.03852396742170358, "cost_per_1M_output_tokens": 0.0, "cost_per_h": 2.39, "cost_source": "RunPod", "cost_additional_info": "Output token cost is disregarded. The cost per 1M input tokens is estimated as total_cost * (1,000,000 / total_input_tokens).", "total_cost": 0.08423947350978853 }, "execution_specifications": { "type": "Local", "details": "This model was ran on an H100 (80GB PCIe) on RunPod using the llm-guard Python SDK." }, "num_samples": 6406, "accuracy_per_technique": { "ascii_art": 1.0, "fsh": 0.5241935483870968, "adversarial_suffixes": 0.008064516129032258, "tokenbreak": 0.38387096774193546, "structural_obfuscation": 0.8935483870967742, "cognitive_psychological": 0.5193548387096775, "low_resource_language": 0.6325411334552102, "encoding_cyphering": 0.5467741935483871, "dap": 0.5161290322580645 } }, "protectai/deepset-prompt-injections/llm-guard": { "model_provider_use_case": "protectai", "dataset_name": "deepset-prompt-injections", "model_name": "llm-guard", "accuracy": 0.40304182509505704, "accuracy_per_category": { "1": 0.40304182509505704 }, "fpr": 0.007518796992481203, "mean_latency": 0.016067717730818918, "latency_ci_95": { "lower": 0.01597017568114084, "upper": 0.016165259780496995, "mean": 0.016067717730818918, "std_dev": 0.001280456416191478, "n": 662 }, "provider": "RunPod", "model_type": "specialized", "model_developer": "ProtectAI", "model_url": "https://protectai.github.io/llm-guard/input_scanners/prompt_injection/", "cost_info": { "cost_per_1M_input_tokens": 0.35880204141487, "cost_per_1M_output_tokens": 0.0, "cost_per_h": 2.39, "cost_source": "RunPod", "cost_additional_info": "Output token cost is disregarded. The cost per 1M input tokens is estimated as total_cost * (1,000,000 / total_input_tokens).", "total_cost": 0.00706167267759641 }, "execution_specifications": { "type": "Local", "details": "This model was ran on an H100 (80GB PCIe) on RunPod using the llm-guard Python SDK." }, "num_samples": 662 }, "protectai/local_datasets-malignant/llm-guard": { "model_provider_use_case": "protectai", "dataset_name": "local_datasets-malignant", "model_name": "llm-guard", "accuracy": 0.8542713567839196, "accuracy_per_category": { "jailbreak": 0.8542713567839196 }, "fpr": 0.002926115581565472, "mean_latency": 0.01572284798969254, "latency_ci_95": { "lower": 0.015594887878410405, "upper": 0.015850808100974677, "mean": 0.01572284798969254, "std_dev": 0.002583535399488985, "n": 1566 }, "provider": "RunPod", "model_type": "specialized", "model_developer": "ProtectAI", "model_url": "https://protectai.github.io/llm-guard/input_scanners/prompt_injection/", "cost_info": { "cost_per_1M_input_tokens": 0.1500186663407153, "cost_per_1M_output_tokens": 0.0, "cost_per_h": 2.39, "cost_source": "RunPod", "cost_additional_info": "Output token cost is disregarded. The cost per 1M input tokens is estimated as total_cost * (1,000,000 / total_input_tokens).", "total_cost": 0.01634625891248385 }, "execution_specifications": { "type": "Local", "details": "This model was ran on an H100 (80GB PCIe) on RunPod using the llm-guard Python SDK." }, "num_samples": 1566 }, "x-ai/TrustAIRLab-in-the-wild-jailbreak-prompts/grok-4-1-fast-non-reasoning": { "model_provider_use_case": "x-ai", "dataset_name": "TrustAIRLab-in-the-wild-jailbreak-prompts", "model_name": "grok-4-1-fast-non-reasoning", "accuracy": 0.7126099706744868, "accuracy_per_category": { "true": 0.7126099706744868 }, "fpr": "N/A", "mean_latency": 0.6349650870124213, "latency_ci_95": { "lower": 0.6018665832083852, "upper": 0.6680635908164574, "mean": 0.6349650870124213, "std_dev": 0.19909469630928459, "n": 139 }, "provider": "xAI", "model_type": "generalist", "model_developer": "X-AI", "model_url": "https://docs.x.ai/docs/models/grok-4-1-fast-non-reasoning", "cost_info": { "cost_per_1M_input_tokens": 0.2, "cost_per_1M_output_tokens": 0.5, "cost_per_h": "N/A", "cost_source": "X-AI API pricing", "cost_additional_info": "", "total_cost": 0.20579930000000002 }, "execution_specifications": { "type": "API", "details": "REST API accessed through a CPU3 RunPod instance at US-KS-2." }, "num_samples": 1364 }, "x-ai/allenai-wildjailbreak/grok-4-1-fast-non-reasoning": { "model_provider_use_case": "x-ai", "dataset_name": "allenai-wildjailbreak", "model_name": "grok-4-1-fast-non-reasoning", "accuracy": 0.8145, "accuracy_per_category": { "adversarial_harmful": 0.8145 }, "fpr": 0.1380952380952381, "mean_latency": 0.6702868053816023, "latency_ci_95": { "lower": 0.6303894307841268, "upper": 0.7101841799790779, "mean": 0.6702868053816023, "std_dev": 0.30261077224672994, "n": 221 }, "provider": "xAI", "model_type": "generalist", "model_developer": "X-AI", "model_url": "https://docs.x.ai/docs/models/grok-4-1-fast-non-reasoning", "cost_info": { "cost_per_1M_input_tokens": 0.2, "cost_per_1M_output_tokens": 0.5, "cost_per_h": "N/A", "cost_source": "X-AI API pricing", "cost_additional_info": "", "total_cost": 0.1456517 }, "execution_specifications": { "type": "API", "details": "REST API accessed through a CPU3 RunPod instance at US-KS-2." }, "num_samples": 2210 }, "x-ai/centrepourlasecuriteia-BET-jailbreak-dataset/grok-4-1-fast-non-reasoning": { "model_provider_use_case": "x-ai", "dataset_name": "centrepourlasecuriteia-BET-jailbreak-dataset", "model_name": "grok-4-1-fast-non-reasoning", "accuracy": 0.9344103000121462, "accuracy_per_category": { "privacy": 0.9312883435582822, "harm to minors": 0.7134146341463414, "integrity & quality violations": 0.9842041312272175, "illegal activities": 0.9865689865689866, "sexual content": 0.9662650602409638, "physical harm": 0.9556886227544911, "cbrn": 0.9526699029126213, "harmful manipulation": 0.9603841536614646, "cyber": 0.9766009852216748, "self-harm": 0.916058394160584 }, "fpr": 0.1722488038277512, "mean_latency": 0.6060841583521424, "latency_ci_95": { "lower": 0.560989780586805, "upper": 0.6511785361174797, "mean": 0.6060841583521424, "std_dev": 0.6925169795691616, "n": 906 }, "provider": "xAI", "model_type": "generalist", "model_developer": "X-AI", "model_url": "https://docs.x.ai/docs/models/grok-4-1-fast-non-reasoning", "cost_info": { "cost_per_1M_input_tokens": 0.2, "cost_per_1M_output_tokens": 0.5, "cost_per_h": "N/A", "cost_source": "X-AI API pricing", "cost_additional_info": "", "total_cost": 0.9003792000000002 }, "execution_specifications": { "type": "API", "details": "REST API accessed through a CPU3 RunPod instance at US-KS-2." }, "num_samples": 9069 }, "x-ai/centrepourlasecuriteia-jailbreak-dataset/grok-4-1-fast-non-reasoning": { "model_provider_use_case": "x-ai", "dataset_name": "centrepourlasecuriteia-jailbreak-dataset", "model_name": "grok-4-1-fast-non-reasoning", "accuracy": 0.7891774105683675, "accuracy_per_category": { "integrity & quality violations": 0.7110754414125201, "physical harm": 0.9135338345864662, "harm to minors": 0.5343137254901961, "illegal activities": 0.9067164179104478, "cyber": 0.9154135338345865, "privacy": 0.8598130841121495, "sexual content": 0.8074766355140187, "cbrn": 0.7052238805970149, "self-harm": 0.7495291902071564, "harmful manipulation": 0.8392523364485981 }, "fpr": 0.11345939933259176, "mean_latency": 0.5547688812017441, "latency_ci_95": { "lower": 0.5317707023780941, "upper": 0.577767060025394, "mean": 0.5547688812017441, "std_dev": 0.2968433759942281, "n": 640 }, "provider": "xAI", "model_type": "generalist", "model_developer": "X-AI", "model_url": "https://docs.x.ai/docs/models/grok-4-1-fast-non-reasoning", "cost_info": { "cost_per_1M_input_tokens": 0.2, "cost_per_1M_output_tokens": 0.5, "cost_per_h": "N/A", "cost_source": "X-AI API pricing", "cost_additional_info": "", "total_cost": 0.7042455000000001 }, "execution_specifications": { "type": "API", "details": "REST API accessed through a CPU3 RunPod instance at US-KS-2." }, "num_samples": 6406, "accuracy_per_technique": { "ascii_art": 0.4241935483870968, "fsh": 0.9209677419354839, "adversarial_suffixes": 0.7903225806451613, "tokenbreak": 0.8290322580645161, "structural_obfuscation": 0.8145161290322581, "cognitive_psychological": 0.7870967741935484, "low_resource_language": 0.7568555758683729, "encoding_cyphering": 0.8790322580645161, "dap": 0.896774193548387 } }, "x-ai/deepset-prompt-injections/grok-4-1-fast-non-reasoning": { "model_provider_use_case": "x-ai", "dataset_name": "deepset-prompt-injections", "model_name": "grok-4-1-fast-non-reasoning", "accuracy": 0.3155893536121673, "accuracy_per_category": { "1": 0.3155893536121673 }, "fpr": 0.017543859649122806, "mean_latency": 0.6507269099906638, "latency_ci_95": { "lower": 0.5877756118889718, "upper": 0.7136782080923559, "mean": 0.6507269099906638, "std_dev": 0.23601820254333813, "n": 54 }, "provider": "xAI", "model_type": "generalist", "model_developer": "X-AI", "model_url": "https://docs.x.ai/docs/models/grok-4-1-fast-non-reasoning", "cost_info": { "cost_per_1M_input_tokens": 0.2, "cost_per_1M_output_tokens": 0.5, "cost_per_h": "N/A", "cost_source": "X-AI API pricing", "cost_additional_info": "", "total_cost": 0.030135000000000006 }, "execution_specifications": { "type": "API", "details": "REST API accessed through a CPU3 RunPod instance at US-KS-2." }, "num_samples": 662 }, "x-ai/local_datasets-malignant/grok-4-1-fast-non-reasoning": { "model_provider_use_case": "x-ai", "dataset_name": "local_datasets-malignant", "model_name": "grok-4-1-fast-non-reasoning", "accuracy": 0.8894472361809045, "accuracy_per_category": { "jailbreak": 0.8894472361809045 }, "fpr": 0.008046817849305048, "mean_latency": 0.8636070127728619, "latency_ci_95": { "lower": 0.684935335003549, "upper": 1.0422786905421748, "mean": 0.8636070127728619, "std_dev": 1.1458511044180395, "n": 158 }, "provider": "xAI", "model_type": "generalist", "model_developer": "X-AI", "model_url": "https://docs.x.ai/docs/models/grok-4-1-fast-non-reasoning", "cost_info": { "cost_per_1M_input_tokens": 0.2, "cost_per_1M_output_tokens": 0.5, "cost_per_h": "N/A", "cost_source": "X-AI API pricing", "cost_additional_info": "", "total_cost": 0.0795021 }, "execution_specifications": { "type": "API", "details": "REST API accessed through a CPU3 RunPod instance at US-KS-2." }, "num_samples": 1566 }, "bells-o/centrepourlasecuriteia-jailbreak-dataset/opencc-jb-escalation": { "model_provider_use_case": "bells-o", "dataset_name": "centrepourlasecuriteia-jailbreak-dataset", "model_name": "opencc-jb-escalation", "accuracy": 0.8947861379956291, "accuracy_per_category": { "integrity & quality violations": 0.9004815409309791, "benign": 0.8787541713014461, "physical harm": 0.9041353383458647, "harm to minors": 0.8823529411764706, "illegal activities": 0.9048507462686567, "cyber": 0.9191729323308271, "privacy": 0.9177570093457944, "sexual content": 0.8785046728971962, "cbrn": 0.8880597014925373, "self-harm": 0.896421845574388, "harmful manipulation": 0.8841121495327103 }, "fpr": "N/A", "mean_latency": 0.12364855052306062, "latency_ci_95": { "lower": 0.12240881867450736, "upper": 0.12488828237161388, "mean": 0.12364855052306062, "std_dev": 0.05062501374367663, "n": 6406 }, "provider": "RunPod", "model_type": "specialized", "model_developer": "CeSIA", "model_url": "https://huggingface.co/centrepourlasecuriteia/opencc-jb-escalation", "cost_info": { "cost_per_1M_input_tokens": 0.0, "cost_per_1M_output_tokens": 0.0, "cost_per_h": 1.49, "cost_source": "RunPod", "cost_additional_info": "Output token cost is disregarded. The cost per 1M input tokens is estimated as total_cost * (1,000,000 / total_input_tokens).", "total_cost": 0.32783833217488395 }, "execution_specifications": { "type": "Local", "details": "This model was ran on an A100 (80GB PCIe) on RunPod using vLLM." }, "num_samples": 6406, "accuracy_per_technique": { "ascii_art": 0.9972222222222222, "adversarial_suffixes": 0.9833333333333333, "fsh": 0.5638888888888889, "tokenbreak": 0.9347222222222222, "structural_obfuscation": 1.0, "cognitive_psychological": 0.8902777777777777, "low_resource_language": 0.6594427244582043, "dap": 1.0, "encoding_cyphering": 1.0 } } }